Release/integrated cd pipeline

.github/workflows/cd.yml

@@ -86,6 +86,22 @@ jobs:
           SOURCE_DIR: './data/raw'
           DEST_DIR: 'source'
 
+      - name: Build utils library
+        run: ./scripts/build-lib.sh utils-0.1-py3-none-any.whl
+
+      - name: Upload library to S3
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: >
+            --delete
+        env:
+          AWS_S3_BUCKET: ${{ env.BUCKET_NAME }}
+          AWS_REGION: ${{ env.REGION }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          SOURCE_DIR: './dist'
+          DEST_DIR: 'lib'
+
       # set up AWS Credentials so that we can update the glue job
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1

Dockerfile

@@ -6,5 +6,6 @@ RUN pip3 install boto
 RUN pip3 install https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.1/en_ner_bc5cdr_md-0.5.1.tar.gz
 RUN pip3 install nltk
 RUN pip3 install python-dotenv
-
-# ENV AWS_REGION ap-southeast-1
+RUN mkdir dist/
+ADD dist dist
+RUN pip3 install ./dist/utilities-0.1-py3-none-any.whl

assets/output.yaml

@@ -19,6 +19,7 @@ Resources:
         --JOB_NAME: job1
         --SOURCE_CRAWLER: glue-jupyter-ws-3SourceCrawler
         --BUCKET_NAME: glue-jupyter-ws-3-bucket
+        --additional-python-modules: s3://glue-jupyter-ws-3-bucket/lib/utils-0.1-py3-none-any.whl
       GlueVersion: '2.0'
       MaxRetries: 0
       AllocatedCapacity: 2
@@ -39,6 +40,7 @@ Resources:
         --JOB_NAME: "job2"
         --SOURCE_CRAWLER: glue-jupyter-ws-3SourceCrawler
         --BUCKET_NAME: glue-jupyter-ws-3-bucket
+        --additional-python-modules: s3://glue-jupyter-ws-3-bucket/lib/utils-0.1-py3-none-any.whl
       GlueVersion: '2.0'
       MaxRetries: 0
       AllocatedCapacity: 2

scripts/build-lib.sh

@@ -0,0 +1,15 @@
+#!/bin/bash 
+
+python3 -m pip install wheel
+python3 setup.py bdist_wheel
+
+export "BUCKET_NAME=$(yq e '.bucket_name' config/properties.yml)"
+for JOB_DIR in ./src/jobs/job*; do
+  JOB_NAME_RAW=$(basename $JOB_DIR)
+  JOB_NAME="$(tr '[:lower:]' '[:upper:]' <<< ${JOB_NAME_RAW:0:1})${JOB_NAME_RAW:1}" # Capitalizing the first letter
+  YAML_PATH=".Resources.Glue${JOB_NAME}.Properties.DefaultArguments"
+  key="--additional-python-modules"
+  value="s3://$BUCKET_NAME/lib/$1"
+  yq_command="yq e '$YAML_PATH.\"$key\" = \"$value\"' -i assets/output.yaml"
+  eval $yq_command
+done

setup.py

@@ -0,0 +1,8 @@
+from setuptools import setup
+
+setup(
+    packages=["utils"],
+    name="utils",
+    version="0.1",
+    install_requires=[]
+)

src/jobs/job1/.env

@@ -1 +1,3 @@
-JOB_NAME=job1
+JOB_NAME=job1
+BUCKET_NAME=glue-jupyter-ws-3-bucket
+SOURCE_CRAWLER=glue-jupyter-ws-3SourceCrawler

src/jobs/job1/notebook.ipynb

@@ -15,7 +15,8 @@
     "from pyspark.context import SparkContext\n",
     "from awsglue.job import Job\n",
     "from awsglue.utils import getResolvedOptions\n",
-    "from pyspark.sql import functions as F"
+    "from pyspark.sql import functions as F\n",
+    "from utils.env import load_env, get_env_or_args"
    ]
   },
   {
@@ -35,42 +36,13 @@
    "execution_count": 3,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "def load_env(environment):\n",
-    "    if environment == \"local\":        \n",
-    "        from dotenv import load_dotenv\n",
-    "        load_dotenv(\".env\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "RELATIVE_PATH_PREFIX=\"./../../..\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_env_or_args(env_name):\n",
-    "    try:\n",
-    "        if os.environ.get(env_name):\n",
-    "            return os.getenv(env_name)\n",
-    "        else:\n",
-    "            return getResolvedOptions(sys.argv,[env_name])[env_name]\n",
-    "    except:\n",
-    "        return ''\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -97,7 +69,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -137,24 +109,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_env_or_args(env_name):\n",
-    "    try:\n",
-    "        if os.environ.get(env_name):\n",
-    "            return os.getenv(env_name)\n",
-    "        else:\n",
-    "            return getResolvedOptions(sys.argv,[env_name])[env_name]\n",
-    "    except:\n",
-    "        return ''\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -167,7 +122,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {

src/jobs/job2/.env

@@ -1 +1,3 @@
-JOB_NAME=job2
+JOB_NAME=job2
+BUCKET_NAME=glue-jupyter-ws-3-bucket
+SOURCE_CRAWLER=glue-jupyter-ws-3SourceCrawler

src/jobs/job2/notebook.ipynb

@@ -16,7 +16,8 @@
     "from awsglue.job import Job\n",
     "from awsglue.utils import getResolvedOptions\n",
     "from pyspark.sql import functions as F\n",
-    "from pyspark.sql.functions import col, lit, current_date, year"
+    "from pyspark.sql.functions import col, lit, current_date, year\n",
+    "from utils.env import load_env, get_env_or_args"
    ]
   },
   {
@@ -36,25 +37,13 @@
    "execution_count": 3,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "def load_env(environment):\n",
-    "    if environment == \"local\":        \n",
-    "        from dotenv import load_dotenv\n",
-    "        load_dotenv(\".env\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [],
    "source": [
     "RELATIVE_PATH_PREFIX=\"./../../..\""
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -73,7 +62,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -90,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -138,24 +127,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_env_or_args(env_name):\n",
-    "    try:\n",
-    "        if os.environ.get(env_name):\n",
-    "            return os.getenv(env_name)\n",
-    "        else:\n",
-    "            return getResolvedOptions(sys.argv,[env_name])[env_name]\n",
-    "    except:\n",
-    "        return ''\n",
-    "    "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -174,33 +146,41 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 9,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "+----------+---------+--------------------+-------------+\n",
+      "|First Name|Last Name|               Email|Year of Birth|\n",
+      "+----------+---------+--------------------+-------------+\n",
+      "|     Rohan|    Gupta|rohan.gupta@wedne...|         1992|\n",
+      "+----------+---------+--------------------+-------------+\n",
+      "\n",
+      "None\n"
+     ]
+    }
+   ],
    "source": [
     "environment = get_env_or_args(\"ENVIRONMENT_NAME\")\n",
     "load_env(environment)\n",
     "local_file_path = f\"{RELATIVE_PATH_PREFIX}/data/raw/sample.csv\"\n",
     "crawler_name = get_env_or_args(\"SOURCE_CRAWLER\")\n",
-    "data = read_data(environment, local_file_path, crawler_name)"
+    "data = read_data(get_env_or_args(\"ENVIRONMENT_NAME\"), local_file_path, crawler_name)\n",
+    "print(data.show())"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "+----------+---------+--------------------+-------------+\n",
-      "|First Name|Last Name|               Email|Year of Birth|\n",
-      "+----------+---------+--------------------+-------------+\n",
-      "|     Rohan|    Gupta|rohan.gupta@wedne...|         1992|\n",
-      "+----------+---------+--------------------+-------------+\n",
-      "\n",
-      "None\n",
       "+----------+---------+--------------------+-------------+-----------+----+\n",
       "|first_name|last_name|               email|year_of_birth|  full_name| age|\n",
       "+----------+---------+--------------------+-------------+-----------+----+\n",
@@ -213,8 +193,6 @@
     }
    ],
    "source": [
-    "print(data.show())\n",
-    "\n",
     "target_bucket = get_env_or_args(\"BUCKET_NAME\")\n",
     "target_directory = f\"{RELATIVE_PATH_PREFIX if environment == 'local' else target_bucket}/landing/{get_env_or_args('JOB_NAME')}\"\n",
     "transformed_data = transform_to_lower_case(data)\n",

utils/env.py

@@ -0,0 +1,17 @@
+import os
+import sys
+from awsglue.utils import getResolvedOptions
+
+def load_env(environment):
+    if environment == "local":        
+        from dotenv import load_dotenv
+        load_dotenv(".env")
+
+def get_env_or_args(env_name):
+    try:
+        if os.environ.get(env_name):
+            return os.getenv(env_name)
+        else:
+            return getResolvedOptions(sys.argv,[env_name])[env_name]
+    except:
+        return ''