IU-ML-Unsupervised-Unit6.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6eb2a04f",
   "metadata": {},
   "source": [
    "# Machine Learning - Unsupervised Learning and Feature Engineering\n",
    "## Unit 6. Automated Feature Generation\n",
    "### Lecturer: Tai Le Quy\n",
    "#### 31.8.2023\n",
    "Thanks Prof. Dr. Christian Müller-Kett for his code."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec3ed8bc",
   "metadata": {},
   "source": [
    "## 1 Automated Feature Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2214ce2",
   "metadata": {},
   "source": [
    "### 1.1 Featuretools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4384bc0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Install the featuretools from the command prompt: version 1.11.1 (30.8.2023)\n",
    "pip install featuretools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a61673c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\woodwork\\__init__.py:23: FutureWarning: Woodwork may not support Python 3.7 in next non-bugfix release.\n",
      "  \"Woodwork may not support Python 3.7 in next non-bugfix release.\", FutureWarning\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\featuretools\\__init__.py:67: FutureWarning: Featuretools may not support Python 3.7 in next non-bugfix release.\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "#%% import libraries\n",
    "import pandas as pd\n",
    "import featuretools as ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fb920460",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% Remove any limit on the number of columns to display \n",
    "pd.options.display.max_columns = None "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9a7234d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% Remove any limit on the number of rows to display\n",
    "pd.options.display.max_rows = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "651dac7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                name         type  dask_compatible  \\\n",
      "0                               last  aggregation            False   \n",
      "1                         num_unique  aggregation             True   \n",
      "2                                sum  aggregation             True   \n",
      "3                      n_most_common  aggregation            False   \n",
      "4                                all  aggregation             True   \n",
      "5                   time_since_first  aggregation            False   \n",
      "6                                std  aggregation             True   \n",
      "7                               skew  aggregation            False   \n",
      "8                            entropy  aggregation            False   \n",
      "9                              count  aggregation             True   \n",
      "10                          num_true  aggregation             True   \n",
      "11                               max  aggregation             True   \n",
      "12                              mode  aggregation            False   \n",
      "13                               min  aggregation             True   \n",
      "14                              mean  aggregation             True   \n",
      "15                      percent_true  aggregation             True   \n",
      "16                             trend  aggregation            False   \n",
      "17                               any  aggregation             True   \n",
      "18                            median  aggregation            False   \n",
      "19                  avg_time_between  aggregation            False   \n",
      "20                             first  aggregation            False   \n",
      "21                   time_since_last  aggregation            False   \n",
      "22                       part_of_day    transform             True   \n",
      "23                       rolling_min    transform            False   \n",
      "24                add_numeric_scalar    transform             True   \n",
      "25                       rolling_std    transform            False   \n",
      "26                    num_characters    transform             True   \n",
      "27                               and    transform             True   \n",
      "28                              isin    transform             True   \n",
      "29                               age    transform             True   \n",
      "30                      is_month_end    transform             True   \n",
      "31                       is_year_end    transform             True   \n",
      "32           subtract_numeric_scalar    transform             True   \n",
      "33                     rolling_count    transform            False   \n",
      "34                     days_in_month    transform             True   \n",
      "35                           cum_min    transform            False   \n",
      "36                            negate    transform             True   \n",
      "37                is_federal_holiday    transform            False   \n",
      "38                    modulo_numeric    transform             True   \n",
      "39      greater_than_equal_to_scalar    transform             True   \n",
      "40                    is_quarter_end    transform             True   \n",
      "41                           quarter    transform             True   \n",
      "42   scalar_subtract_numeric_feature    transform             True   \n",
      "43                           is_null    transform             True   \n",
      "44                             month    transform             True   \n",
      "45                           cum_max    transform            False   \n",
      "46                     url_to_domain    transform            False   \n",
      "47                              hour    transform             True   \n",
      "48                 modulo_by_feature    transform             True   \n",
      "49                  less_than_scalar    transform             True   \n",
      "50                        percentile    transform            False   \n",
      "51                            second    transform             True   \n",
      "52                  is_quarter_start    transform             True   \n",
      "53           multiply_numeric_scalar    transform             True   \n",
      "54               greater_than_scalar    transform             True   \n",
      "55                          absolute    transform             True   \n",
      "56                         haversine    transform            False   \n",
      "57                      greater_than    transform             True   \n",
      "58                               not    transform             True   \n",
      "59                       day_of_year    transform             True   \n",
      "60             greater_than_equal_to    transform             True   \n",
      "61                  multiply_numeric    transform             True   \n",
      "62                              week    transform             True   \n",
      "63                               day    transform             True   \n",
      "64             divide_numeric_scalar    transform             True   \n",
      "65                       square_root    transform             True   \n",
      "66                     is_year_start    transform             True   \n",
      "67                   url_to_protocol    transform            False   \n",
      "68                        is_weekend    transform             True   \n",
      "69          multiply_numeric_boolean    transform             True   \n",
      "70                less_than_equal_to    transform             True   \n",
      "71                       geomidpoint    transform            False   \n",
      "72                   date_to_holiday    transform            False   \n",
      "73                    divide_numeric    transform             True   \n",
      "74                      equal_scalar    transform             True   \n",
      "75                 natural_logarithm    transform             True   \n",
      "76                          latitude    transform            False   \n",
      "77                                or    transform             True   \n",
      "78                        url_to_tld    transform            False   \n",
      "79                    is_month_start    transform             True   \n",
      "80             modulo_numeric_scalar    transform             True   \n",
      "81         less_than_equal_to_scalar    transform             True   \n",
      "82                         less_than    transform             True   \n",
      "83                              diff    transform            False   \n",
      "84                         longitude    transform            False   \n",
      "85                            minute    transform             True   \n",
      "86                 divide_by_feature    transform             True   \n",
      "87                              sine    transform             True   \n",
      "88                      is_in_geobox    transform            False   \n",
      "89                           cum_sum    transform            False   \n",
      "90           email_address_to_domain    transform            False   \n",
      "91                           weekday    transform             True   \n",
      "92               time_since_previous    transform            False   \n",
      "93                         not_equal    transform             True   \n",
      "94              is_free_email_domain    transform            False   \n",
      "95                       rolling_max    transform            False   \n",
      "96                              year    transform             True   \n",
      "97                  multiply_boolean    transform             True   \n",
      "98                           tangent    transform             True   \n",
      "99               distance_to_holiday    transform            False   \n",
      "100                            equal    transform             True   \n",
      "101                      numeric_lag    transform            False   \n",
      "102               cityblock_distance    transform            False   \n",
      "103                     is_leap_year    transform             True   \n",
      "104                      add_numeric    transform             True   \n",
      "105                 subtract_numeric    transform             True   \n",
      "106                     rolling_mean    transform            False   \n",
      "107                           cosine    transform             True   \n",
      "108                         cum_mean    transform            False   \n",
      "109                        num_words    transform             True   \n",
      "110                       time_since    transform             True   \n",
      "111                        cum_count    transform            False   \n",
      "112                 not_equal_scalar    transform             True   \n",
      "\n",
      "     spark_compatible                                        description  \\\n",
      "0               False               Determines the last value in a list.   \n",
      "1                True  Determines the number of distinct values, igno...   \n",
      "2                True     Calculates the total addition, ignoring `NaN`.   \n",
      "3               False           Determines the `n` most common elements.   \n",
      "4               False     Calculates if all values are 'True' in a list.   \n",
      "5               False  Calculates the time elapsed since the first da...   \n",
      "6                True  Computes the dispersion relative to the mean v...   \n",
      "7               False  Computes the extent to which a distribution di...   \n",
      "8               False    Calculates the entropy for a categorical column   \n",
      "9                True  Determines the total number of values, excludi...   \n",
      "10              False                Counts the number of `True` values.   \n",
      "11               True  Calculates the highest value, ignoring `NaN` v...   \n",
      "12              False       Determines the most commonly repeated value.   \n",
      "13               True  Calculates the smallest value, ignoring `NaN` ...   \n",
      "14               True         Computes the average for a list of values.   \n",
      "15              False           Determines the percent of `True` values.   \n",
      "16              False        Calculates the trend of a column over time.   \n",
      "17              False       Determines if any value is 'True' in a list.   \n",
      "18              False  Determines the middlemost number in a list of ...   \n",
      "19              False  Computes the average number of seconds between...   \n",
      "20              False              Determines the first value in a list.   \n",
      "21              False  Calculates the time elapsed since the last dat...   \n",
      "22               True          Determines the part of day of a datetime.   \n",
      "23              False  Determines the minimum of entries over a given...   \n",
      "24               True            Add a scalar to each value in the list.   \n",
      "25              False  Calculates the standard deviation of entries o...   \n",
      "26               True   Calculates the number of characters in a string.   \n",
      "27               True             Element-wise logical AND of two lists.   \n",
      "28               True  Determines whether a value is present in a pro...   \n",
      "29              False  Calculates the age in years as a floating poin...   \n",
      "30               True  Determines the is_month_end attribute of a dat...   \n",
      "31               True   Determines if a date falls on the end of a year.   \n",
      "32               True   Subtract a scalar from each element in the list.   \n",
      "33              False  Determines a rolling count of events over a gi...   \n",
      "34               True   Determines the day of the month from a datetime.   \n",
      "35              False                 Calculates the cumulative minimum.   \n",
      "36               True                           Negates a numeric value.   \n",
      "37              False  Determines if a given datetime is a federal ho...   \n",
      "38               True                  Element-wise modulo of two lists.   \n",
      "39               True  Determines if values are greater than or equal...   \n",
      "40               True  Determines the is_quarter_end attribute of a d...   \n",
      "41               True  Determines the quarter a datetime column falls...   \n",
      "42               True  Subtract each value in the list from a given s...   \n",
      "43               True                     Determines if a value is null.   \n",
      "44               True          Determines the month value of a datetime.   \n",
      "45              False                 Calculates the cumulative maximum.   \n",
      "46              False                    Determines the domain of a url.   \n",
      "47               True           Determines the hour value of a datetime.   \n",
      "48               True  Return the modulo of a scalar by each element ...   \n",
      "49               True  Determines if values are less than a given sca...   \n",
      "50              False  Determines the percentile rank for each value ...   \n",
      "51               True        Determines the seconds value of a datetime.   \n",
      "52               True  Determines the is_quarter_start attribute of a...   \n",
      "53               True     Multiply each element in the list by a scalar.   \n",
      "54               True  Determines if values are greater than a given ...   \n",
      "55               True           Computes the absolute value of a number.   \n",
      "56              False  Calculates the approximate haversine distance ...   \n",
      "57              False  Determines if values in one list are greater t...   \n",
      "58               True                           Negates a boolean value.   \n",
      "59               True  Determines the ordinal day of the year from th...   \n",
      "60               True  Determines if values in one list are greater t...   \n",
      "61               True          Element-wise multiplication of two lists.   \n",
      "62               True   Determines the week of the year from a datetime.   \n",
      "63               True   Determines the day of the month from a datetime.   \n",
      "64               True       Divide each element in the list by a scalar.   \n",
      "65               True              Computes the square root of a number.   \n",
      "66               True  Determines if a date falls on the start of a y...   \n",
      "67              False  Determines the protocol (http or https) of a url.   \n",
      "68               True           Determines if a date falls on a weekend.   \n",
      "69              False  Element-wise multiplication of a numeric list ...   \n",
      "70               True  Determines if values in one list are less than...   \n",
      "71              False  Determines the geographic center of two coordi...   \n",
      "72              False  Transforms time of an instance into the holida...   \n",
      "73               True                Element-wise division of two lists.   \n",
      "74               True  Determines if values in a list are equal to a ...   \n",
      "75               True        Computes the natural logarithm of a number.   \n",
      "76              False  Returns the first tuple value in a list of Lat...   \n",
      "77               True              Element-wise logical OR of two lists.   \n",
      "78              False          Determines the top level domain of a url.   \n",
      "79               True  Determines the is_month_start attribute of a d...   \n",
      "80               True  Return the modulo of each element in the list ...   \n",
      "81               True  Determines if values are less than or equal to...   \n",
      "82               True  Determines if values in one list are less than...   \n",
      "83              False  Compute the difference between the value in a ...   \n",
      "84              False  Returns the second tuple value in a list of La...   \n",
      "85               True        Determines the minutes value of a datetime.   \n",
      "86               True         Divide a scalar by each value in the list.   \n",
      "87               True                     Computes the sine of a number.   \n",
      "88              False  Determines if coordinates are inside a box def...   \n",
      "89              False                     Calculates the cumulative sum.   \n",
      "90              False                  Determines the domain of an email   \n",
      "91               True    Determines the day of the week from a datetime.   \n",
      "92              False  Compute the time since the previous entry in a...   \n",
      "93              False  Determines if values in one list are not equal...   \n",
      "94              False  Determines if an email address is from a free ...   \n",
      "95              False  Determines the maximum of entries over a given...   \n",
      "96               True           Determines the year value of a datetime.   \n",
      "97              False  Element-wise multiplication of two lists of bo...   \n",
      "98               True                  Computes the tangent of a number.   \n",
      "99              False  Computes the number of days before or after a ...   \n",
      "100              True  Determines if values in one list are equal to ...   \n",
      "101             False  Shifts an array of values by a specified numbe...   \n",
      "102             False  Calculates the distance between points in a ci...   \n",
      "103              True  Determines the is_leap_year attribute of a dat...   \n",
      "104              True                Element-wise addition of two lists.   \n",
      "105             False             Element-wise subtraction of two lists.   \n",
      "106             False  Calculates the mean of entries over a given wi...   \n",
      "107              True                   Computes the cosine of a number.   \n",
      "108             False                    Calculates the cumulative mean.   \n",
      "109              True  Determines the number of words in a string by ...   \n",
      "110             False  Calculates time from a value to a specified cu...   \n",
      "111             False                   Calculates the cumulative count.   \n",
      "112              True  Determines if values in a list are not equal t...   \n",
      "\n",
      "                                          valid_inputs  \\\n",
      "0                                       <ColumnSchema>   \n",
      "1        <ColumnSchema (Semantic Tags = ['category'])>   \n",
      "2         <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "3        <ColumnSchema (Semantic Tags = ['category'])>   \n",
      "4    <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "5    <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "6         <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "7         <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "8        <ColumnSchema (Semantic Tags = ['category'])>   \n",
      "9           <ColumnSchema (Semantic Tags = ['index'])>   \n",
      "10   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "11        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "12       <ColumnSchema (Semantic Tags = ['category'])>   \n",
      "13        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "14        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "15   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "16   <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "17   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "18        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "19   <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "20                                      <ColumnSchema>   \n",
      "21   <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "22            <ColumnSchema (Logical Type = Datetime)>   \n",
      "23   <ColumnSchema (Semantic Tags = ['numeric'])>, ...   \n",
      "24        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "25   <ColumnSchema (Semantic Tags = ['numeric'])>, ...   \n",
      "26     <ColumnSchema (Logical Type = NaturalLanguage)>   \n",
      "27   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "28                                      <ColumnSchema>   \n",
      "29   <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "30            <ColumnSchema (Logical Type = Datetime)>   \n",
      "31            <ColumnSchema (Logical Type = Datetime)>   \n",
      "32        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "33   <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "34            <ColumnSchema (Logical Type = Datetime)>   \n",
      "35        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "36        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "37            <ColumnSchema (Logical Type = Datetime)>   \n",
      "38        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "39   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "40            <ColumnSchema (Logical Type = Datetime)>   \n",
      "41            <ColumnSchema (Logical Type = Datetime)>   \n",
      "42        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "43                                      <ColumnSchema>   \n",
      "44            <ColumnSchema (Logical Type = Datetime)>   \n",
      "45        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "46                 <ColumnSchema (Logical Type = URL)>   \n",
      "47            <ColumnSchema (Logical Type = Datetime)>   \n",
      "48        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "49   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "50        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "51            <ColumnSchema (Logical Type = Datetime)>   \n",
      "52            <ColumnSchema (Logical Type = Datetime)>   \n",
      "53        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "54   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "55        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "56             <ColumnSchema (Logical Type = LatLong)>   \n",
      "57   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "58   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "59            <ColumnSchema (Logical Type = Datetime)>   \n",
      "60   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "61        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "62            <ColumnSchema (Logical Type = Datetime)>   \n",
      "63            <ColumnSchema (Logical Type = Datetime)>   \n",
      "64        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "65        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "66            <ColumnSchema (Logical Type = Datetime)>   \n",
      "67                 <ColumnSchema (Logical Type = URL)>   \n",
      "68            <ColumnSchema (Logical Type = Datetime)>   \n",
      "69   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "70   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "71             <ColumnSchema (Logical Type = LatLong)>   \n",
      "72            <ColumnSchema (Logical Type = Datetime)>   \n",
      "73        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "74                                      <ColumnSchema>   \n",
      "75        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "76             <ColumnSchema (Logical Type = LatLong)>   \n",
      "77   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "78                 <ColumnSchema (Logical Type = URL)>   \n",
      "79            <ColumnSchema (Logical Type = Datetime)>   \n",
      "80        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "81   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "82   <ColumnSchema (Logical Type = Datetime)>, <Col...   \n",
      "83        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "84             <ColumnSchema (Logical Type = LatLong)>   \n",
      "85            <ColumnSchema (Logical Type = Datetime)>   \n",
      "86        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "87        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "88             <ColumnSchema (Logical Type = LatLong)>   \n",
      "89        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "90        <ColumnSchema (Logical Type = EmailAddress)>   \n",
      "91            <ColumnSchema (Logical Type = Datetime)>   \n",
      "92   <ColumnSchema (Logical Type = Datetime) (Seman...   \n",
      "93                                      <ColumnSchema>   \n",
      "94        <ColumnSchema (Logical Type = EmailAddress)>   \n",
      "95   <ColumnSchema (Semantic Tags = ['numeric'])>, ...   \n",
      "96            <ColumnSchema (Logical Type = Datetime)>   \n",
      "97   <ColumnSchema (Logical Type = BooleanNullable)...   \n",
      "98        <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "99            <ColumnSchema (Logical Type = Datetime)>   \n",
      "100                                     <ColumnSchema>   \n",
      "101  <ColumnSchema (Semantic Tags = ['time_index'])...   \n",
      "102            <ColumnSchema (Logical Type = LatLong)>   \n",
      "103           <ColumnSchema (Logical Type = Datetime)>   \n",
      "104       <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "105       <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "106  <ColumnSchema (Semantic Tags = ['numeric'])>, ...   \n",
      "107       <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "108       <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
      "109    <ColumnSchema (Logical Type = NaturalLanguage)>   \n",
      "110           <ColumnSchema (Logical Type = Datetime)>   \n",
      "111  <ColumnSchema (Semantic Tags = ['foreign_key']...   \n",
      "112                                     <ColumnSchema>   \n",
      "\n",
      "                                           return_type  \n",
      "0                                                 None  \n",
      "1    <ColumnSchema (Logical Type = IntegerNullable)...  \n",
      "2         <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "3                                                 None  \n",
      "4              <ColumnSchema (Logical Type = Boolean)>  \n",
      "5    <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "6         <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "7         <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "8         <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "9    <ColumnSchema (Logical Type = IntegerNullable)...  \n",
      "10   <ColumnSchema (Logical Type = IntegerNullable)...  \n",
      "11        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "12                                                None  \n",
      "13        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "14        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "15   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "16        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "17             <ColumnSchema (Logical Type = Boolean)>  \n",
      "18        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "19   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "20                                                None  \n",
      "21   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "22   <ColumnSchema (Logical Type = Categorical) (Se...  \n",
      "23   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "24        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "25   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "26        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "27     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "28             <ColumnSchema (Logical Type = Boolean)>  \n",
      "29   <ColumnSchema (Logical Type = AgeFractional) (...  \n",
      "30     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "31     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "32        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "33   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "34   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "35        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "36        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "37     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "38        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "39     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "40     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "41   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "42        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "43             <ColumnSchema (Logical Type = Boolean)>  \n",
      "44   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "45        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "46   <ColumnSchema (Logical Type = Categorical) (Se...  \n",
      "47   <ColumnSchema (Logical Type = Ordinal: [0, 1, ...  \n",
      "48        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "49     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "50        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "51   <ColumnSchema (Logical Type = Ordinal: [0, 1, ...  \n",
      "52     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "53        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "54     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "55        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "56        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "57     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "58     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "59   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "60     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "61        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "62   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "63   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "64        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "65   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "66     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "67   <ColumnSchema (Logical Type = Categorical) (Se...  \n",
      "68     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "69        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "70     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "71             <ColumnSchema (Logical Type = LatLong)>  \n",
      "72   <ColumnSchema (Logical Type = Categorical) (Se...  \n",
      "73        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "74     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "75   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "76        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "77     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "78   <ColumnSchema (Logical Type = Categorical) (Se...  \n",
      "79     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "80        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "81     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "82     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "83        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "84        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "85   <ColumnSchema (Logical Type = Ordinal: [0, 1, ...  \n",
      "86        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "87   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "88     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "89        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "90   <ColumnSchema (Logical Type = Categorical) (Se...  \n",
      "91   <ColumnSchema (Logical Type = Ordinal: [0, 1, ...  \n",
      "92        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "93     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "94     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "95   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "96   <ColumnSchema (Logical Type = Ordinal: [1, 2, ...  \n",
      "97     <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "98   <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "99        <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "100    <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "101       <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "102  <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "103    <ColumnSchema (Logical Type = BooleanNullable)>  \n",
      "104       <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "105       <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "106  <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "107  <ColumnSchema (Logical Type = Double) (Semanti...  \n",
      "108       <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "109       <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "110       <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
      "111  <ColumnSchema (Logical Type = IntegerNullable)...  \n",
      "112    <ColumnSchema (Logical Type = BooleanNullable)>  \n"
     ]
    }
   ],
   "source": [
    "#%% Display the list of primitives\n",
    "print(ft.list_primitives())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ea95f57",
   "metadata": {},
   "source": [
    "### 1.2 Automatically generate new features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "650e0d6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\woodwork\\__init__.py:23: FutureWarning: Woodwork may not support Python 3.7 in next non-bugfix release.\n",
      "  \"Woodwork may not support Python 3.7 in next non-bugfix release.\", FutureWarning\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\featuretools\\__init__.py:67: FutureWarning: Featuretools may not support Python 3.7 in next non-bugfix release.\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "import featuretools as ft\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5865b48c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% create sample data\n",
    "Customers = pd.DataFrame({ \n",
    "    'C_ID': ['C1', 'C2'], \n",
    "    'Name': ['Martin', 'Julia'], \n",
    "    'Creation_date': ['2018-08-15', '2020-05-05']}, \n",
    "        columns = ['C_ID','Name','Creation_date'])\n",
    "Orders = pd.DataFrame({ \n",
    "    'Ord_ID': ['1', '2', '3', '4', '5'], \n",
    "    'C_ID': ['C1', 'C2', 'C1', 'C1','C2']}, \n",
    "        columns = ['Ord_ID','C_ID'])\n",
    "Payments = pd.DataFrame({ \n",
    "    'Ord_ID':['1', '5', '3', '4', '2'], \n",
    "    'Price':[500, 200, 300, 100, 900]}, \n",
    "        columns = ['Ord_ID', 'Price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "92fddcde",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C_ID</th>\n",
       "      <th>Name</th>\n",
       "      <th>Creation_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C1</td>\n",
       "      <td>Martin</td>\n",
       "      <td>2018-08-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C2</td>\n",
       "      <td>Julia</td>\n",
       "      <td>2020-05-05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  C_ID    Name Creation_date\n",
       "0   C1  Martin    2018-08-15\n",
       "1   C2   Julia    2020-05-05"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Customers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1239f587",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ord_ID</th>\n",
       "      <th>C_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>C2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>C2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Ord_ID C_ID\n",
       "0      1   C1\n",
       "1      2   C2\n",
       "2      3   C1\n",
       "3      4   C1\n",
       "4      5   C2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Orders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4c6d414f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ord_ID</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Ord_ID  Price\n",
       "0      1    500\n",
       "1      5    200\n",
       "2      3    300\n",
       "3      4    100\n",
       "4      2    900"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Payments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "12635346",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% create 'customer' entitysets\n",
    "es = ft.EntitySet(id = 'Customers')\n",
    "es = es.add_dataframe( \\\n",
    "    dataframe_name = 'Customers', \\\n",
    "    dataframe = Customers, \\\n",
    "    index = 'C_ID', time_index = 'Creation_date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "83ff68b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: Customers\n",
       "  DataFrames:\n",
       "    Customers [Rows: 2, Columns: 3]\n",
       "  Relationships:\n",
       "    No relationships"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9d7afdfd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% create orders entityset\n",
    "es = es.add_dataframe( \\\n",
    "    dataframe_name = 'Orders', \\\n",
    "    dataframe = Orders, \\\n",
    "    index = 'Ord_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3cc1a75b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% create payments entityset\n",
    "es = es.add_dataframe( \\\n",
    "    dataframe_name = 'Payments', \\\n",
    "    dataframe = Payments, \n",
    "    make_index = True,\n",
    "    index = 'P_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1f7be61a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>P_ID</th>\n",
       "      <th>Ord_ID</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   P_ID Ord_ID  Price\n",
       "0     0      1    500\n",
       "1     1      5    200\n",
       "2     2      3    300\n",
       "3     3      4    100\n",
       "4     4      2    900"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es['Payments']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "27931ade",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: Customers\n",
       "  DataFrames:\n",
       "    Customers [Rows: 2, Columns: 3]\n",
       "    Orders [Rows: 5, Columns: 2]\n",
       "    Payments [Rows: 5, Columns: 3]\n",
       "  Relationships:\n",
       "    No relationships"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "17625e48",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%\n",
    "# Define the relationship between the parent 'Customers' \n",
    "# and the child 'Orders' linked together by 'C_ID'\n",
    "r_Cust_Ord = ft.Relationship(entityset=es,parent_dataframe_name= 'Customers', parent_column_name='C_ID', \n",
    "                             child_dataframe_name= 'Orders',child_column_name='C_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0c23af09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Relationship: Orders.C_ID -> Customers.C_ID>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_Cust_Ord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fe63334e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% Add the relationship to the entity set\n",
    "es = es.add_relationship(relationship=r_Cust_Ord)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1960c32b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: Customers\n",
       "  DataFrames:\n",
       "    Customers [Rows: 2, Columns: 3]\n",
       "    Orders [Rows: 5, Columns: 2]\n",
       "    Payments [Rows: 5, Columns: 3]\n",
       "  Relationships:\n",
       "    Orders.C_ID -> Customers.C_ID"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e9fad5ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% define relationship between 'Orders' \n",
    "# and 'Payments'\n",
    "r_Orders_Payments = ft.Relationship(entityset=es,parent_dataframe_name= 'Orders', \n",
    "                                    parent_column_name='Ord_ID',child_dataframe_name= 'Payments',\n",
    "                                    child_column_name='Ord_ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "68a04ad5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% Add the relationship to the entity set\n",
    "es = es.add_relationship(relationship=r_Orders_Payments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4a4bf30e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: Customers\n",
       "  DataFrames:\n",
       "    Customers [Rows: 2, Columns: 3]\n",
       "    Orders [Rows: 5, Columns: 2]\n",
       "    Payments [Rows: 5, Columns: 3]\n",
       "  Relationships:\n",
       "    Orders.C_ID -> Customers.C_ID\n",
       "    Payments.Ord_ID -> Orders.Ord_ID"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% show entityset\n",
    "es"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "968d0806",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>type</th>\n",
       "      <th>dask_compatible</th>\n",
       "      <th>spark_compatible</th>\n",
       "      <th>description</th>\n",
       "      <th>valid_inputs</th>\n",
       "      <th>return_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>all</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates if all values are 'True' in a list.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Boolean)&gt;, &lt;ColumnSchema (Logical Type = BooleanNullable)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Boolean)&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mode</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines the most commonly repeated value.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['category'])&gt;</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>trend</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates the trend of a column over time.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;, &lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>std</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Computes the dispersion relative to the mean value, ignoring `NaN`.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>entropy</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates the entropy for a categorical column</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['category'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>time_since_last</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates the time elapsed since the last datetime (default in seconds).</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>median</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines the middlemost number in a list of values.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>avg_time_between</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Computes the average number of seconds between consecutive events.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>n_most_common</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines the `n` most common elements.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['category'])&gt;</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>percent_true</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines the percent of `True` values.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Boolean)&gt;, &lt;ColumnSchema (Logical Type = BooleanNullable)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>any</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines if any value is 'True' in a list.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Boolean)&gt;, &lt;ColumnSchema (Logical Type = BooleanNullable)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Boolean)&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Calculates the highest value, ignoring `NaN` values.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>first</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines the first value in a list.</td>\n",
       "      <td>&lt;ColumnSchema&gt;</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>count</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Determines the total number of values, excluding `NaN`.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = IntegerNullable) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>mean</td>\n",
       "      <td>aggregation</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Computes the average for a list of values.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                name         type  dask_compatible  spark_compatible  \\\n",
       "0                all  aggregation             True             False   \n",
       "1               mode  aggregation            False             False   \n",
       "2              trend  aggregation            False             False   \n",
       "3                std  aggregation             True              True   \n",
       "4            entropy  aggregation            False             False   \n",
       "5    time_since_last  aggregation            False             False   \n",
       "6             median  aggregation            False             False   \n",
       "7   avg_time_between  aggregation            False             False   \n",
       "8      n_most_common  aggregation            False             False   \n",
       "9       percent_true  aggregation             True             False   \n",
       "10               any  aggregation             True             False   \n",
       "11               max  aggregation             True              True   \n",
       "12             first  aggregation            False             False   \n",
       "13             count  aggregation             True              True   \n",
       "14              mean  aggregation             True              True   \n",
       "\n",
       "                                                                  description  \\\n",
       "0                              Calculates if all values are 'True' in a list.   \n",
       "1                                Determines the most commonly repeated value.   \n",
       "2                                 Calculates the trend of a column over time.   \n",
       "3         Computes the dispersion relative to the mean value, ignoring `NaN`.   \n",
       "4                             Calculates the entropy for a categorical column   \n",
       "5   Calculates the time elapsed since the last datetime (default in seconds).   \n",
       "6                       Determines the middlemost number in a list of values.   \n",
       "7          Computes the average number of seconds between consecutive events.   \n",
       "8                                    Determines the `n` most common elements.   \n",
       "9                                    Determines the percent of `True` values.   \n",
       "10                               Determines if any value is 'True' in a list.   \n",
       "11                       Calculates the highest value, ignoring `NaN` values.   \n",
       "12                                      Determines the first value in a list.   \n",
       "13                    Determines the total number of values, excluding `NaN`.   \n",
       "14                                 Computes the average for a list of values.   \n",
       "\n",
       "                                                                                                               valid_inputs  \\\n",
       "0                                  <ColumnSchema (Logical Type = Boolean)>, <ColumnSchema (Logical Type = BooleanNullable)>   \n",
       "1                                                                             <ColumnSchema (Semantic Tags = ['category'])>   \n",
       "2   <ColumnSchema (Semantic Tags = ['numeric'])>, <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])>   \n",
       "3                                                                              <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "4                                                                             <ColumnSchema (Semantic Tags = ['category'])>   \n",
       "5                                                 <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])>   \n",
       "6                                                                              <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "7                                                 <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])>   \n",
       "8                                                                             <ColumnSchema (Semantic Tags = ['category'])>   \n",
       "9                                  <ColumnSchema (Logical Type = Boolean)>, <ColumnSchema (Logical Type = BooleanNullable)>   \n",
       "10                                 <ColumnSchema (Logical Type = Boolean)>, <ColumnSchema (Logical Type = BooleanNullable)>   \n",
       "11                                                                             <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "12                                                                                                           <ColumnSchema>   \n",
       "13                                                                               <ColumnSchema (Semantic Tags = ['index'])>   \n",
       "14                                                                             <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "\n",
       "                                                                      return_type  \n",
       "0                                         <ColumnSchema (Logical Type = Boolean)>  \n",
       "1                                                                            None  \n",
       "2                                    <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "3                                    <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "4                                    <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "5            <ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])>  \n",
       "6                                    <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "7            <ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])>  \n",
       "8                                                                            None  \n",
       "9            <ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])>  \n",
       "10                                        <ColumnSchema (Logical Type = Boolean)>  \n",
       "11                                   <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "12                                                                           None  \n",
       "13  <ColumnSchema (Logical Type = IntegerNullable) (Semantic Tags = ['numeric'])>  \n",
       "14                                   <ColumnSchema (Semantic Tags = ['numeric'])>  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% show aggregation primitives\n",
    "primitives = ft.list_primitives()\n",
    "pd.options.display.max_colwidth = 160\n",
    "primitives[primitives['type']==\"aggregation\"].head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "eca898f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>type</th>\n",
       "      <th>dask_compatible</th>\n",
       "      <th>spark_compatible</th>\n",
       "      <th>description</th>\n",
       "      <th>valid_inputs</th>\n",
       "      <th>return_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>part_of_day</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Determines the part of day of a datetime.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Categorical) (Semantic Tags = ['category'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>rolling_min</td>\n",
       "      <td>transform</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines the minimum of entries over a given window.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;, &lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>add_numeric_scalar</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Add a scalar to each value in the list.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>rolling_std</td>\n",
       "      <td>transform</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates the standard deviation of entries over a given window.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;, &lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>num_characters</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Calculates the number of characters in a string.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = NaturalLanguage)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>and</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Element-wise logical AND of two lists.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = BooleanNullable)&gt;, &lt;ColumnSchema (Logical Type = Boolean)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = BooleanNullable)&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>isin</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Determines whether a value is present in a provided list.</td>\n",
       "      <td>&lt;ColumnSchema&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Boolean)&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>age</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates the age in years as a floating point number given a</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['date_of_birth'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = AgeFractional) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>is_month_end</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Determines the is_month_end attribute of a datetime column.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = BooleanNullable)&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>is_year_end</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Determines if a date falls on the end of a year.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = BooleanNullable)&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>subtract_numeric_scalar</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Subtract a scalar from each element in the list.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>rolling_count</td>\n",
       "      <td>transform</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Determines a rolling count of events over a given window.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>days_in_month</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Determines the day of the month from a datetime.</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Datetime)&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Logical Type = Ordinal: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]) ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>cum_min</td>\n",
       "      <td>transform</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Calculates the cumulative minimum.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>negate</td>\n",
       "      <td>transform</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Negates a numeric value.</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "      <td>&lt;ColumnSchema (Semantic Tags = ['numeric'])&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name       type  dask_compatible  spark_compatible  \\\n",
       "22              part_of_day  transform             True              True   \n",
       "23              rolling_min  transform            False             False   \n",
       "24       add_numeric_scalar  transform             True              True   \n",
       "25              rolling_std  transform            False             False   \n",
       "26           num_characters  transform             True              True   \n",
       "27                      and  transform             True              True   \n",
       "28                     isin  transform             True              True   \n",
       "29                      age  transform             True             False   \n",
       "30             is_month_end  transform             True              True   \n",
       "31              is_year_end  transform             True              True   \n",
       "32  subtract_numeric_scalar  transform             True              True   \n",
       "33            rolling_count  transform            False             False   \n",
       "34            days_in_month  transform             True              True   \n",
       "35                  cum_min  transform            False             False   \n",
       "36                   negate  transform             True              True   \n",
       "\n",
       "                                                          description  \\\n",
       "22                          Determines the part of day of a datetime.   \n",
       "23             Determines the minimum of entries over a given window.   \n",
       "24                            Add a scalar to each value in the list.   \n",
       "25  Calculates the standard deviation of entries over a given window.   \n",
       "26                   Calculates the number of characters in a string.   \n",
       "27                             Element-wise logical AND of two lists.   \n",
       "28          Determines whether a value is present in a provided list.   \n",
       "29     Calculates the age in years as a floating point number given a   \n",
       "30        Determines the is_month_end attribute of a datetime column.   \n",
       "31                   Determines if a date falls on the end of a year.   \n",
       "32                   Subtract a scalar from each element in the list.   \n",
       "33          Determines a rolling count of events over a given window.   \n",
       "34                   Determines the day of the month from a datetime.   \n",
       "35                                 Calculates the cumulative minimum.   \n",
       "36                                           Negates a numeric value.   \n",
       "\n",
       "                                                                                                               valid_inputs  \\\n",
       "22                                                                                 <ColumnSchema (Logical Type = Datetime)>   \n",
       "23  <ColumnSchema (Semantic Tags = ['numeric'])>, <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])>   \n",
       "24                                                                             <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "25  <ColumnSchema (Semantic Tags = ['numeric'])>, <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])>   \n",
       "26                                                                          <ColumnSchema (Logical Type = NaturalLanguage)>   \n",
       "27                                 <ColumnSchema (Logical Type = BooleanNullable)>, <ColumnSchema (Logical Type = Boolean)>   \n",
       "28                                                                                                           <ColumnSchema>   \n",
       "29                                             <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['date_of_birth'])>   \n",
       "30                                                                                 <ColumnSchema (Logical Type = Datetime)>   \n",
       "31                                                                                 <ColumnSchema (Logical Type = Datetime)>   \n",
       "32                                                                             <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "33                                                <ColumnSchema (Logical Type = Datetime) (Semantic Tags = ['time_index'])>   \n",
       "34                                                                                 <ColumnSchema (Logical Type = Datetime)>   \n",
       "35                                                                             <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "36                                                                             <ColumnSchema (Semantic Tags = ['numeric'])>   \n",
       "\n",
       "                                                                                                                                                        return_type  \n",
       "22                                                                                       <ColumnSchema (Logical Type = Categorical) (Semantic Tags = ['category'])>  \n",
       "23                                                                                             <ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])>  \n",
       "24                                                                                                                     <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "25                                                                                             <ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])>  \n",
       "26                                                                                                                     <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "27                                                                                                                  <ColumnSchema (Logical Type = BooleanNullable)>  \n",
       "28                                                                                                                          <ColumnSchema (Logical Type = Boolean)>  \n",
       "29                                                                                      <ColumnSchema (Logical Type = AgeFractional) (Semantic Tags = ['numeric'])>  \n",
       "30                                                                                                                  <ColumnSchema (Logical Type = BooleanNullable)>  \n",
       "31                                                                                                                  <ColumnSchema (Logical Type = BooleanNullable)>  \n",
       "32                                                                                                                     <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "33                                                                                             <ColumnSchema (Logical Type = Double) (Semantic Tags = ['numeric'])>  \n",
       "34  <ColumnSchema (Logical Type = Ordinal: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]) ...  \n",
       "35                                                                                                                     <ColumnSchema (Semantic Tags = ['numeric'])>  \n",
       "36                                                                                                                     <ColumnSchema (Semantic Tags = ['numeric'])>  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#%% show transformation primitives\n",
    "primitives[primitives['type']==\"transform\"].head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "cf5d78f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% generate features\n",
    "features, feature_names = ft.dfs( \n",
    "    entityset=es, \n",
    "    target_dataframe_name='Customers', \n",
    "    agg_primitives=['sum'], \n",
    "    trans_primitives=['year'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b6f55d56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUM(Payments.Price)</th>\n",
       "      <th>YEAR(Creation_date)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1</th>\n",
       "      <td>900.0</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C2</th>\n",
       "      <td>1100.0</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      SUM(Payments.Price) YEAR(Creation_date)\n",
       "C_ID                                         \n",
       "C1                  900.0                2018\n",
       "C2                 1100.0                2020"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b3c04ef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% generate features\n",
    "feats, feat_names = ft.dfs( entityset=es, \n",
    "    target_dataframe_name='Customers', \n",
    "    max_depth = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "900de71e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>COUNT(Orders)</th>\n",
       "      <th>COUNT(Payments)</th>\n",
       "      <th>MAX(Payments.Price)</th>\n",
       "      <th>MEAN(Payments.Price)</th>\n",
       "      <th>MIN(Payments.Price)</th>\n",
       "      <th>SKEW(Payments.Price)</th>\n",
       "      <th>STD(Payments.Price)</th>\n",
       "      <th>SUM(Payments.Price)</th>\n",
       "      <th>DAY(Creation_date)</th>\n",
       "      <th>MONTH(Creation_date)</th>\n",
       "      <th>...</th>\n",
       "      <th>STD(Orders.MAX(Payments.Price))</th>\n",
       "      <th>STD(Orders.MEAN(Payments.Price))</th>\n",
       "      <th>STD(Orders.MIN(Payments.Price))</th>\n",
       "      <th>STD(Orders.SKEW(Payments.Price))</th>\n",
       "      <th>STD(Orders.SUM(Payments.Price))</th>\n",
       "      <th>SUM(Orders.MAX(Payments.Price))</th>\n",
       "      <th>SUM(Orders.MEAN(Payments.Price))</th>\n",
       "      <th>SUM(Orders.MIN(Payments.Price))</th>\n",
       "      <th>SUM(Orders.SKEW(Payments.Price))</th>\n",
       "      <th>SUM(Orders.STD(Payments.Price))</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>200.000000</td>\n",
       "      <td>900.0</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>200.000000</td>\n",
       "      <td>200.000000</td>\n",
       "      <td>200.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200.000000</td>\n",
       "      <td>900.0</td>\n",
       "      <td>900.0</td>\n",
       "      <td>900.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>900.0</td>\n",
       "      <td>550.0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>494.974747</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>494.974747</td>\n",
       "      <td>494.974747</td>\n",
       "      <td>494.974747</td>\n",
       "      <td>NaN</td>\n",
       "      <td>494.974747</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      COUNT(Orders)  COUNT(Payments)  MAX(Payments.Price)  \\\n",
       "C_ID                                                        \n",
       "C1                3                3                500.0   \n",
       "C2                2                2                900.0   \n",
       "\n",
       "      MEAN(Payments.Price)  MIN(Payments.Price)  SKEW(Payments.Price)  \\\n",
       "C_ID                                                                    \n",
       "C1                   300.0                100.0                   0.0   \n",
       "C2                   550.0                200.0                   NaN   \n",
       "\n",
       "      STD(Payments.Price)  SUM(Payments.Price) DAY(Creation_date)  \\\n",
       "C_ID                                                                \n",
       "C1             200.000000                900.0                 15   \n",
       "C2             494.974747               1100.0                  5   \n",
       "\n",
       "     MONTH(Creation_date)  ... STD(Orders.MAX(Payments.Price))  \\\n",
       "C_ID                       ...                                   \n",
       "C1                      8  ...                      200.000000   \n",
       "C2                      5  ...                      494.974747   \n",
       "\n",
       "     STD(Orders.MEAN(Payments.Price))  STD(Orders.MIN(Payments.Price))  \\\n",
       "C_ID                                                                     \n",
       "C1                         200.000000                       200.000000   \n",
       "C2                         494.974747                       494.974747   \n",
       "\n",
       "      STD(Orders.SKEW(Payments.Price))  STD(Orders.SUM(Payments.Price))  \\\n",
       "C_ID                                                                      \n",
       "C1                                 NaN                       200.000000   \n",
       "C2                                 NaN                       494.974747   \n",
       "\n",
       "      SUM(Orders.MAX(Payments.Price))  SUM(Orders.MEAN(Payments.Price))  \\\n",
       "C_ID                                                                      \n",
       "C1                              900.0                             900.0   \n",
       "C2                             1100.0                            1100.0   \n",
       "\n",
       "      SUM(Orders.MIN(Payments.Price))  SUM(Orders.SKEW(Payments.Price))  \\\n",
       "C_ID                                                                      \n",
       "C1                              900.0                               0.0   \n",
       "C2                             1100.0                               0.0   \n",
       "\n",
       "      SUM(Orders.STD(Payments.Price))  \n",
       "C_ID                                   \n",
       "C1                                0.0  \n",
       "C2                                0.0  \n",
       "\n",
       "[2 rows x 48 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6fa02ff5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<Feature: COUNT(Orders)>,\n",
       " <Feature: COUNT(Payments)>,\n",
       " <Feature: MAX(Payments.Price)>,\n",
       " <Feature: MEAN(Payments.Price)>,\n",
       " <Feature: MIN(Payments.Price)>,\n",
       " <Feature: SKEW(Payments.Price)>,\n",
       " <Feature: STD(Payments.Price)>,\n",
       " <Feature: SUM(Payments.Price)>,\n",
       " <Feature: DAY(Creation_date)>,\n",
       " <Feature: MONTH(Creation_date)>,\n",
       " <Feature: WEEKDAY(Creation_date)>,\n",
       " <Feature: YEAR(Creation_date)>,\n",
       " <Feature: MAX(Orders.COUNT(Payments))>,\n",
       " <Feature: MAX(Orders.MEAN(Payments.Price))>,\n",
       " <Feature: MAX(Orders.MIN(Payments.Price))>,\n",
       " <Feature: MAX(Orders.SKEW(Payments.Price))>,\n",
       " <Feature: MAX(Orders.STD(Payments.Price))>,\n",
       " <Feature: MAX(Orders.SUM(Payments.Price))>,\n",
       " <Feature: MEAN(Orders.COUNT(Payments))>,\n",
       " <Feature: MEAN(Orders.MAX(Payments.Price))>,\n",
       " <Feature: MEAN(Orders.MEAN(Payments.Price))>,\n",
       " <Feature: MEAN(Orders.MIN(Payments.Price))>,\n",
       " <Feature: MEAN(Orders.SKEW(Payments.Price))>,\n",
       " <Feature: MEAN(Orders.STD(Payments.Price))>,\n",
       " <Feature: MEAN(Orders.SUM(Payments.Price))>,\n",
       " <Feature: MIN(Orders.COUNT(Payments))>,\n",
       " <Feature: MIN(Orders.MAX(Payments.Price))>,\n",
       " <Feature: MIN(Orders.MEAN(Payments.Price))>,\n",
       " <Feature: MIN(Orders.SKEW(Payments.Price))>,\n",
       " <Feature: MIN(Orders.STD(Payments.Price))>,\n",
       " <Feature: MIN(Orders.SUM(Payments.Price))>,\n",
       " <Feature: SKEW(Orders.COUNT(Payments))>,\n",
       " <Feature: SKEW(Orders.MAX(Payments.Price))>,\n",
       " <Feature: SKEW(Orders.MEAN(Payments.Price))>,\n",
       " <Feature: SKEW(Orders.MIN(Payments.Price))>,\n",
       " <Feature: SKEW(Orders.STD(Payments.Price))>,\n",
       " <Feature: SKEW(Orders.SUM(Payments.Price))>,\n",
       " <Feature: STD(Orders.COUNT(Payments))>,\n",
       " <Feature: STD(Orders.MAX(Payments.Price))>,\n",
       " <Feature: STD(Orders.MEAN(Payments.Price))>,\n",
       " <Feature: STD(Orders.MIN(Payments.Price))>,\n",
       " <Feature: STD(Orders.SKEW(Payments.Price))>,\n",
       " <Feature: STD(Orders.SUM(Payments.Price))>,\n",
       " <Feature: SUM(Orders.MAX(Payments.Price))>,\n",
       " <Feature: SUM(Orders.MEAN(Payments.Price))>,\n",
       " <Feature: SUM(Orders.MIN(Payments.Price))>,\n",
       " <Feature: SUM(Orders.SKEW(Payments.Price))>,\n",
       " <Feature: SUM(Orders.STD(Payments.Price))>]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7d94a6e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}