Added travis config for CI/CD.

chcost · chcost · commit 5e9612e26bbc · 2021-06-12T19:16:40.000-04:00
diff --git a/.travis.yml b/.travis.yml
@@ -14,15 +14,15 @@
 # limitations under the License.
 #
 
-dist: bionic
 language: python
-services:
-  - docker
+python:
+  - "3.8"
+  - "3.9"
+  - "3.9-dev"  # 3.9 development branch
+  - "nightly"  # nightly build
 
-#before_install:
-#  - ./scripts/travis.sh
+install:
+  - pip install -r requirements.txt
 
-#install:
-#  - pyenv global 3.8.1
-#  - pip install --upgrade pip
-#  - pip install .
+script:
+  - pytest
diff --git a/README.md b/README.md
@@ -32,9 +32,35 @@ pip install --upgrade pip
 pip install .
 ```
 
-### Try a first example
+### Example notebooks
 
-**TODO:** Add instructions for running the notebooks in the `notebooks` directory.
+You can find a selection of examples in [notebooks](./notebooks). As a first example, we recommend the [Sample Pipeline](./notebooks/sample_pipeline.ipynb
+
+**Running locally**
+
+To run the example pipelines, you will need the following requirements:
+- Jupyter Jupyter
+- Ray
+- sklearn
+- pandas
+- pytest
+- numpy
+- pickle51
+- graphviz
+
+They can be installed with:
+
+If you use pip, you can install it with:
+```shell
+pip install -r requirements.txt 
+```
+
+Run the sample pipeline with:
+```shell
+jupyter-lab sample_pipeline.ipynb
+```
+
+The pipeline will use `ray.init()` to start a local Ray cluster. See [configuring Ray](https://docs.ray.io/en/master/configure.html) to ensure you are able to run a Ray cluster locally.
 
 ## Contributing
 
diff --git a/notebooks/sample_pipeline.ipynb b/notebooks/sample_pipeline.ipynb
@@ -12,12 +12,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "id": "simplified-summit",
    "metadata": {},
    "outputs": [
     {
-     "output_type": "execute_result",
      "data": {
       "text/plain": [
        "Gender                object\n",
@@ -35,8 +34,9 @@
        "dtype: object"
       ]
      },
+     "execution_count": 2,
      "metadata": {},
-     "execution_count": 1
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -49,19 +49,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "id": "configured-clinton",
    "metadata": {},
    "outputs": [
     {
-     "output_type": "execute_result",
      "data": {
       "text/plain": [
        "(614, 12)"
       ]
      },
+     "execution_count": 3,
      "metadata": {},
-     "execution_count": 2
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -130,10 +130,10 @@
    "metadata": {},
    "outputs": [
     {
-     "output_type": "stream",
      "name": "stdout",
+     "output_type": "stream",
      "text": [
-      "Time taken: 0.014804840087890625\n"
+      "Time taken: 0.010515928268432617\n"
      ]
     }
    ],
@@ -230,14 +230,14 @@
    "metadata": {},
    "outputs": [
     {
-     "output_type": "execute_result",
      "data": {
       "text/plain": [
        "KNeighborsClassifier(n_neighbors=3)"
       ]
      },
+     "execution_count": 12,
      "metadata": {},
-     "execution_count": 12
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -274,14 +274,14 @@
    "metadata": {},
    "outputs": [
     {
-     "output_type": "execute_result",
      "data": {
       "text/plain": [
        "True"
       ]
      },
+     "execution_count": 15,
      "metadata": {},
-     "execution_count": 15
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -295,14 +295,14 @@
    "metadata": {},
    "outputs": [
     {
-     "output_type": "execute_result",
      "data": {
       "text/plain": [
        "ScaleTestEstimator(classifier=DecisionTreeClassifier(), num_iters=50)"
       ]
      },
+     "execution_count": 16,
      "metadata": {},
-     "execution_count": 16
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -332,10 +332,10 @@
    "metadata": {},
    "outputs": [
     {
-     "output_type": "stream",
      "name": "stdout",
+     "output_type": "stream",
      "text": [
-      "time taken: 9.157356023788452\n"
+      "time taken: 9.11965799331665\n"
      ]
     }
    ],
@@ -369,11 +369,11 @@
     {
      "data": {
       "text/plain": [
-       "array([0.18937852, 0.11576017, 0.15804519, 0.0422178 , 0.29690848,\n",
-       "       0.        , 0.01586572, 0.0056865 , 0.01781055, 0.00389469,\n",
-       "       0.        , 0.02261156, 0.03051274, 0.00307078, 0.00307078,\n",
-       "       0.00690926, 0.01138089, 0.00499624, 0.02322422, 0.00829843,\n",
-       "       0.01728413, 0.01025423, 0.01281911, 0.        ])"
+       "array([0.25963757, 0.12413346, 0.15603435, 0.02163518, 0.26895302,\n",
+       "       0.02420672, 0.        , 0.00735991, 0.01356057, 0.02023574,\n",
+       "       0.        , 0.016999  , 0.00676174, 0.00954828, 0.00981587,\n",
+       "       0.        , 0.01254879, 0.00851478, 0.01445153, 0.        ,\n",
+       "       0.        , 0.00044522, 0.01758959, 0.00756869])"
       ]
      },
      "execution_count": 18,
@@ -387,33 +387,47 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 19,
    "id": "coordinate-gossip",
    "metadata": {},
-   "outputs": [
-    {
-     "output_type": "error",
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'ray'",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "\u001b[0;32m<ipython-input-18-b94c8db5a623>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'ray'"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import ray\n",
     "ray.shutdown()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "id": "bfe20bd8",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2021-06-12 19:04:59,104\tINFO services.py:1267 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'node_ip_address': '192.168.86.160',\n",
+       " 'raylet_ip_address': '192.168.86.160',\n",
+       " 'redis_address': '192.168.86.160:6379',\n",
+       " 'object_store_address': '/tmp/ray/session_2021-06-12_19-04-57_727692_23228/sockets/plasma_store',\n",
+       " 'raylet_socket_name': '/tmp/ray/session_2021-06-12_19-04-57_727692_23228/sockets/raylet',\n",
+       " 'webui_url': '127.0.0.1:8265',\n",
+       " 'session_dir': '/tmp/ray/session_2021-06-12_19-04-57_727692_23228',\n",
+       " 'metrics_export_port': 57599,\n",
+       " 'node_id': '2942ae78a25dfdfabdde6aee3dd586a83644c3b3c51615cc864d7c93'}"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "ray.init()"
    ]
@@ -506,7 +520,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Time taken: 9.811173915863037\n"
+      "Time taken: 7.180877923965454\n"
      ]
     }
    ],
@@ -532,7 +546,7 @@
     {
      "data": {
       "text/plain": [
-       "[<codeflare.pipelines.Datamodel.XYRef at 0x7f95d8b9f550>]"
+       "[<codeflare.pipelines.Datamodel.XYRef at 0x186fe9c10>]"
       ]
      },
      "execution_count": 29,
@@ -564,8 +578,28 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "preprocess=\r\n",
-      "c_a=preprocess \r\n",
+      "preprocess{'n_jobs': None, 'remainder': 'drop', 'sparse_threshold': 0.3, 'transformer_weights': None, 'transformers': [('num', Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),\n",
+      "                ('scaler', StandardScaler())]), Index(['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',\n",
+      "       'Loan_Amount_Term', 'Credit_History'],\n",
+      "      dtype='object')), ('cat', Pipeline(steps=[('imputer',\n",
+      "                 SimpleImputer(fill_value='missing', strategy='constant')),\n",
+      "                ('onehot', OneHotEncoder(handle_unknown='ignore'))]), Index(['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed',\n",
+      "       'Property_Area'],\n",
+      "      dtype='object'))], 'verbose': False, 'num': Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),\n",
+      "                ('scaler', StandardScaler())]), 'cat': Pipeline(steps=[('imputer',\n",
+      "                 SimpleImputer(fill_value='missing', strategy='constant')),\n",
+      "                ('onehot', OneHotEncoder(handle_unknown='ignore'))]), 'num__memory': None, 'num__steps': [('imputer', SimpleImputer(strategy='median')), ('scaler', StandardScaler())], 'num__verbose': False, 'num__imputer': SimpleImputer(strategy='median'), 'num__scaler': StandardScaler(), 'num__imputer__add_indicator': False, 'num__imputer__copy': True, 'num__imputer__fill_value': None, 'num__imputer__missing_values': nan, 'num__imputer__strategy': 'median', 'num__imputer__verbose': 0, 'num__scaler__copy': True, 'num__scaler__with_mean': True, 'num__scaler__with_std': True, 'cat__memory': None, 'cat__steps': [('imputer', SimpleImputer(fill_value='missing', strategy='constant')), ('onehot', OneHotEncoder(handle_unknown='ignore'))], 'cat__verbose': False, 'cat__imputer': SimpleImputer(fill_value='missing', strategy='constant'), 'cat__onehot': OneHotEncoder(handle_unknown='ignore'), 'cat__imputer__add_indicator': False, 'cat__imputer__copy': True, 'cat__imputer__fill_value': 'missing', 'cat__imputer__missing_values': nan, 'cat__imputer__strategy': 'constant', 'cat__imputer__verbose': 0, 'cat__onehot__categories': 'auto', 'cat__onehot__drop': None, 'cat__onehot__dtype': <class 'numpy.float64'>, 'cat__onehot__handle_unknown': 'ignore', 'cat__onehot__sparse': True}=\n",
+      "c_a{'classifier__ccp_alpha': 0.0, 'classifier__class_weight': None, 'classifier__criterion': 'gini', 'classifier__max_depth': None, 'classifier__max_features': None, 'classifier__max_leaf_nodes': None, 'classifier__min_impurity_decrease': 0.0, 'classifier__min_impurity_split': None, 'classifier__min_samples_leaf': 1, 'classifier__min_samples_split': 2, 'classifier__min_weight_fraction_leaf': 0.0, 'classifier__random_state': None, 'classifier__splitter': 'best', 'classifier': DecisionTreeClassifier(), 'num_iters': 50}=preprocess{'n_jobs': None, 'remainder': 'drop', 'sparse_threshold': 0.3, 'transformer_weights': None, 'transformers': [('num', Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),\n",
+      "                ('scaler', StandardScaler())]), Index(['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount',\n",
+      "       'Loan_Amount_Term', 'Credit_History'],\n",
+      "      dtype='object')), ('cat', Pipeline(steps=[('imputer',\n",
+      "                 SimpleImputer(fill_value='missing', strategy='constant')),\n",
+      "                ('onehot', OneHotEncoder(handle_unknown='ignore'))]), Index(['Gender', 'Married', 'Dependents', 'Education', 'Self_Employed',\n",
+      "       'Property_Area'],\n",
+      "      dtype='object'))], 'verbose': False, 'num': Pipeline(steps=[('imputer', SimpleImputer(strategy='median')),\n",
+      "                ('scaler', StandardScaler())]), 'cat': Pipeline(steps=[('imputer',\n",
+      "                 SimpleImputer(fill_value='missing', strategy='constant')),\n",
+      "                ('onehot', OneHotEncoder(handle_unknown='ignore'))]), 'num__memory': None, 'num__steps': [('imputer', SimpleImputer(strategy='median')), ('scaler', StandardScaler())], 'num__verbose': False, 'num__imputer': SimpleImputer(strategy='median'), 'num__scaler': StandardScaler(), 'num__imputer__add_indicator': False, 'num__imputer__copy': True, 'num__imputer__fill_value': None, 'num__imputer__missing_values': nan, 'num__imputer__strategy': 'median', 'num__imputer__verbose': 0, 'num__scaler__copy': True, 'num__scaler__with_mean': True, 'num__scaler__with_std': True, 'cat__memory': None, 'cat__steps': [('imputer', SimpleImputer(fill_value='missing', strategy='constant')), ('onehot', OneHotEncoder(handle_unknown='ignore'))], 'cat__verbose': False, 'cat__imputer': SimpleImputer(fill_value='missing', strategy='constant'), 'cat__onehot': OneHotEncoder(handle_unknown='ignore'), 'cat__imputer__add_indicator': False, 'cat__imputer__copy': True, 'cat__imputer__fill_value': 'missing', 'cat__imputer__missing_values': nan, 'cat__imputer__strategy': 'constant', 'cat__imputer__verbose': 0, 'cat__onehot__categories': 'auto', 'cat__onehot__drop': None, 'cat__onehot__dtype': <class 'numpy.float64'>, 'cat__onehot__handle_unknown': 'ignore', 'cat__onehot__sparse': True} \n",
       "\n"
      ]
     }
@@ -584,9 +618,13 @@
   }
  ],
  "metadata": {
+  "interpreter": {
+   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
+  },
   "kernelspec": {
-   "name": "python3",
-   "display_name": "Python 3.9.2 64-bit"
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -598,12 +636,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.2"
-  },
-  "interpreter": {
-   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
+   "version": "3.8.6"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}
