Curried_functions_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sys, os\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'function': 'count_letter',\n",
       "  'source': 'word_1',\n",
       "  'letter': 't',\n",
       "  'target': 't_count_1'},\n",
       " {'function': 'count_letter',\n",
       "  'source': 'word_2',\n",
       "  'letter': 'e',\n",
       "  'target': 'e_count_2'},\n",
       " {'function': 'sum_columns',\n",
       "  'sources': ['t_count_1', 'e_count_2'],\n",
       "  'target': 't1_e2_total'},\n",
       " {'function': 'rename_column',\n",
       "  'source': 't1_e2_total',\n",
       "  'target': 'letter_count'},\n",
       " {'function': 'drop_columns', 'columns_to_drop': ['t_count_1', 'e_count_2']}]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define our pipeline using a JSON file\n",
    "pipeline_json = \"\"\"\n",
    "    [\n",
    "    {\"function\":\"count_letter\", \"source\":\"word_1\", \"letter\":\"t\", \"target\":\"t_count_1\"},\n",
    "    {\"function\":\"count_letter\", \"source\":\"word_2\", \"letter\":\"e\", \"target\":\"e_count_2\"},\n",
    "    {\"function\":\"sum_columns\", \"sources\":[\"t_count_1\", \"e_count_2\"], \"target\":\"t1_e2_total\"},\n",
    "    {\"function\":\"rename_column\", \"source\":\"t1_e2_total\", \"target\":\"letter_count\"},\n",
    "    {\"function\":\"drop_columns\", \"columns_to_drop\":[\"t_count_1\", \"e_count_2\"]}\n",
    "    ]\n",
    "    \"\"\"\n",
    "\n",
    "# convert the string to JSON\n",
    "pipeline_definition = json.loads(pipeline_json)\n",
    "\n",
    "# export this definition to a text file to simulate the way we would actually want to import the data\n",
    "with open('pipeline_file.txt', 'w') as outfile:\n",
    "    json.dump(pipeline_definition, outfile)\n",
    "\n",
    "# read from the text file to recover our pipeline in JSON format (note json.load NOT json.loads)\n",
    "with open(\"pipeline_file.txt\") as pipeline_file:\n",
    "    pipeline_definition = json.load(pipeline_file)\n",
    "    \n",
    "pipeline_definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets build a dataframe\n",
    "data = {\n",
    "    \"word_1\": [\"terrible\", \"totter\", \"banana\", \"thick\", \"transparent\"],\n",
    "    \"word_2\": [\"excellent\", \"strident\", \"oblong\", \"regular\", \"three\"]\n",
    "       }\n",
    "\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# export the dataframe to a csv to allow us to simulate importing data\n",
    "df.to_csv(\"df.csv\", index = False)\n",
    "\n",
    "# import the csv to get our dataframe\n",
    "df = pd.read_csv(\"df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word_1</th>\n",
       "      <th>word_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>terrible</td>\n",
       "      <td>excellent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>totter</td>\n",
       "      <td>strident</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>banana</td>\n",
       "      <td>oblong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thick</td>\n",
       "      <td>regular</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>transparent</td>\n",
       "      <td>three</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word_1     word_2\n",
       "0     terrible  excellent\n",
       "1       totter   strident\n",
       "2       banana     oblong\n",
       "3        thick    regular\n",
       "4  transparent      three"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the standard functions we want to be able to use\n",
    "\n",
    "class DataFrameHandler():\n",
    "    \n",
    "    def __init__(self):\n",
    "        # initialise an empty pipeline\n",
    "        self.function_pipeline = []\n",
    "    \n",
    "    def rename_column(self, spec):\n",
    "        '''function to rename a column'''\n",
    "        # check if this function curries the requested function\n",
    "        if spec[\"function\"] != \"rename_column\":\n",
    "            # if it was not, then return None\n",
    "            return None\n",
    "        # otherwise extract the variables to be curried into this function\n",
    "        source = spec[\"source\"]\n",
    "        target = spec[\"target\"]\n",
    "        # then curry the function which wil carry out the requested action\n",
    "        def fn(df):\n",
    "            #curried renaming function\n",
    "            try:\n",
    "                df = df.rename(columns={source: target})\n",
    "            except:\n",
    "                print(\"unable to complete rename_column\")\n",
    "            return df\n",
    "        # return the internal function fn to be used in the pipeline\n",
    "        return fn\n",
    "    \n",
    "    def sum_columns(self, spec):\n",
    "        '''function to sum a list of columns'''\n",
    "        if spec[\"function\"] != \"sum_columns\":\n",
    "            return None\n",
    "        sources = spec[\"sources\"]\n",
    "        target = spec[\"target\"]\n",
    "        def fn(df):\n",
    "            #curried sum function\n",
    "            try:\n",
    "                df[target] = df[sources].sum(axis=1)\n",
    "            except:\n",
    "                print(\"unable to complete sum_columns\")\n",
    "            return df\n",
    "        return fn\n",
    "    \n",
    "    def count_letter(self, spec):\n",
    "        '''function to create letter frequency counts'''\n",
    "        if spec[\"function\"] != \"count_letter\":\n",
    "            return None\n",
    "        source = spec[\"source\"]\n",
    "        target = spec[\"target\"]\n",
    "        letter = spec[\"letter\"]\n",
    "        # curry in a helper function to be used on the dataframe using apply\n",
    "        def _letter_tally(phrase):\n",
    "            count = 0\n",
    "            for l in phrase:\n",
    "                #print(\"letter = \" + l)\n",
    "                if l == letter:\n",
    "                    count += 1\n",
    "            return count\n",
    "        def fn(df):\n",
    "            # curried count letter function\n",
    "            try:\n",
    "                df[target] = df[source].apply(_letter_tally)\n",
    "            except:\n",
    "                print(\"unable to complete count_letter\")\n",
    "            return df\n",
    "        return fn\n",
    "    \n",
    "    def drop_columns(self, spec):\n",
    "        '''function to delete a given list of columns from the dataframe'''\n",
    "        if spec[\"function\"] != \"drop_columns\":\n",
    "            return None\n",
    "        columns_to_drop = spec[\"columns_to_drop\"]\n",
    "        def fn(df):\n",
    "            # curried drop columns function\n",
    "            try:\n",
    "                df = df.drop(columns_to_drop, axis = 1)\n",
    "            except:\n",
    "                print(\"unable to complete drop_columns\")\n",
    "            return df\n",
    "        return fn\n",
    "        \n",
    "    def create_pipeline(self, pipeline_definition):\n",
    "        '''create a pipeline from a definition'''\n",
    "        # enumerate the available pipeline functions\n",
    "        available_functions = [self.rename_column, \n",
    "                               self.sum_columns, \n",
    "                               self.count_letter, \n",
    "                               self.drop_columns]\n",
    "        # step through the spec seeking the correct function for each operation\n",
    "        for spec in pipeline_definition:\n",
    "            for func in available_functions:\n",
    "                if func(spec) is not None:\n",
    "                    # add the desired functions to the pipeline, currying in the required variables\n",
    "                    self.function_pipeline.append(func(spec))\n",
    "                    continue\n",
    "                    \n",
    "    def clear_pipeline(self):\n",
    "        '''removes current pipeline functions'''\n",
    "        self.function_pipeline = []\n",
    "                    \n",
    "    def run_pipeline(self, df):\n",
    "        '''apply the pipeline functions in order to the dataframe'''\n",
    "        for func in self.function_pipeline:\n",
    "            df = func(df)\n",
    "        return df     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word_1</th>\n",
       "      <th>word_2</th>\n",
       "      <th>letter_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>terrible</td>\n",
       "      <td>excellent</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>totter</td>\n",
       "      <td>strident</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>banana</td>\n",
       "      <td>oblong</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thick</td>\n",
       "      <td>regular</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>transparent</td>\n",
       "      <td>three</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        word_1     word_2  letter_count\n",
       "0     terrible  excellent             4\n",
       "1       totter   strident             4\n",
       "2       banana     oblong             0\n",
       "3        thick    regular             2\n",
       "4  transparent      three             4"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create our pipeline object\n",
    "pipeline_object = DataFrameHandler()\n",
    "# add the correct functions as per the defintion\n",
    "pipeline_object.create_pipeline(pipeline_definition)\n",
    "# run the pipeline on our data\n",
    "output_df = pipeline_object.run_pipeline(df)\n",
    "output_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}