Merge pull request #92 from datmo/0.0.2-pypi

0.0.2 pypi changes and upload

datmo/VERSION

@@ -1 +1 @@
-0.0.2-dev
+0.0.3-dev

datmo/core/util/logger.py

@@ -2,6 +2,7 @@
 
 import hashlib
 import logging
+import logging.handlers
 import os
 import tempfile
 import time
@@ -10,8 +11,8 @@
 
 class DatmoLogger(object):
     """ Datmo Logging singleton
-    Good info https://fangpenlin.com/posts/2012/08/26/good-logging-practice-in-python/
 
+    Good info https://fangpenlin.com/posts/2012/08/26/good-logging-practice-in-python/
     """
 
     instance = None
@@ -75,16 +76,14 @@ def get_logger(name=__name__, file_name="log.txt"):
         Parameters
         ----------
         name : string
-          Name of the logger.  eg datmo.core.snapshot or __name__
-
+            Name of the logger.  eg datmo.core.snapshot or __name__
         file_name : string
-          Log file name. default = log.txt
+            Log file name. default = log.txt
 
         Returns
         -------
         logging.Logger
-          Python logger
-
+            Python logger
         """
 
         # get or create logging path

datmo/core/util/tests/test_logger.py

@@ -12,21 +12,21 @@
 class TestLogger():
     def test_logger(self):
         logger = DatmoLogger.get_logger()
-        logger.warn("testing")
+        logger.warning("testing")
         logger.error("WHAT?")
         assert logger.level == DatmoLogger().logging_level
 
     def test_datmo_logger_get_logfiles(self):
         if hasattr(logging, 'handlers'):
             files = DatmoLogger.get_logfiles()
-            assert len(files) > 0
+            assert len(list(files)) > 0
             for f in files:
                 assert DatmoLogger().logging_path in f
 
     def test_find_text_in_logs(self):
         if hasattr(logging, 'handlers'):
             logger = DatmoLogger.get_logger()
-            logger.warn("can you find this")
+            logger.warning("can you find this")
             results = DatmoLogger.find_text_in_logs("can you find this")
             assert len(results) == 1
             for r in results:
@@ -43,17 +43,17 @@ def test_multiple_loggers(self):
         if hasattr(logging, 'handlers'):
             l1 = DatmoLogger.get_logger("logger1")
             l2 = DatmoLogger.get_logger("logger2")
-            l1.warn("pizza")
-            l2.warn("party")
+            l1.warning("pizza")
+            l2.warning("party")
             assert len(DatmoLogger.find_text_in_logs("pizza")) == 1
             assert len(DatmoLogger.find_text_in_logs("logger2")) == 1
 
     def test_multiple_log_files(self):
         if hasattr(logging, 'handlers'):
             l1 = DatmoLogger.get_logger("logger3", "debug.txt")
             l2 = DatmoLogger.get_logger("logger3", "info.txt")
-            l1.warn("green")
-            l2.warn("purple")
+            l1.warning("green")
+            l2.warning("purple")
             r = DatmoLogger.find_text_in_logs("green")
             assert len(r) == 1
             assert r[0]["file"].find("debug.txt")
@@ -67,10 +67,10 @@ def test_autocreate_dir(self):
             shutil.rmtree(log_path)
             assert not os.path.exists(log_path)
             logger = DatmoLogger.get_logger("logger3", "new.txt")
-            logger.warn("testing")
+            logger.warning("testing")
             assert len(DatmoLogger.find_text_in_logs("testing")) == 1
             default_logger = DatmoLogger.get_logger()
-            default_logger.warn("default-logger")
+            default_logger.warning("default-logger")
             assert len(DatmoLogger.find_text_in_logs("default-logger")) == 1
 
     def test_timeit_decorator(self):

devtools/README.md

@@ -22,10 +22,12 @@ $ pip install recommonmark
 ```
 
 ## Testing
+The testing done below emulates what is done in the build on [travis](https://travis-ci.org/datmo/datmo)
 ```
 $ pip install pytest pytest-cov
 $ pip install coveralls
 $ export TEST_DATMO_DIR="/mydir" # Must be an absolute path
+$ export LOGGING_LEVEL=DEBUG # sets logging level to debug for tests
 $ python -m pytest --cov-config .coveragerc --cov=datmo
 ```
 
@@ -49,4 +51,14 @@ else
     yapf -i $CHANGED_FILES
     git diff --name-only --cached | xargs -l git add
 fi
-```
+```
+
+## Upload to PyPi
+Versions of datmo are uploaded to [PyPI](https://pypi.org/project/datmo/) with the following steps
+1) Ensure the `VERSION` file in the `datmo/` module is updated to the desired version to upload 
+(must be later than the previous version on PyPI based on semver)
+2) Run the following commands
+
+        $ rm -rf dist/
+        $ python setup.py bdist_wheel
+        $ twine upload dist/*

examples/README.md

@@ -1,47 +1,87 @@
 # Examples
 
 In order to run the examples, make sure that you have datmo properly installed with the latest 
-stable or development version.
+stable or development version. You can install it with the following command:
+
+```
+$ pip install datmo
+```
 
 ## Using the Examples
 ### CLI
-1. `$ pip install datmo` or `$ git clone https://github.com/datmo/datmo`
-2. Navigate to desired project folder
-3. Copy/save example files within project folder
-4. Open the `SCRIPT_NAME.sh` file in a text editor and run the commands individually, or run `$ bash SCRIPT_NAME.sh`
+1. Navigate to desired project folder or create a new one 
+
+        $ mkdir MY_DATMO_PROJECT
+        $ cd MY_DATMO_PROJECT
+
+2. Copy/save example files within project folder
+
+        $ cp /path/to/SCRIPT_NAME.sh .
+        $ cp /path/to/SCRIPT_NAME.py .
+
+3. Open the `SCRIPT_NAME.sh` file in a text editor and run the commands individually, or run
+
+        $ bash SCRIPT_NAME.sh
 
 ### Python
-1. `$ pip install datmo`
-2. Navigate to desired project folder
-3. `$ datmo init`
-4. Copy/save example files within project folder
-5. Run `python SCRIPT_NAME.py`
+1. Navigate to desired project folder or create a new one 
+
+        $ mkdir MY_DATMO_PROJECT
+        $ cd MY_DATMO_PROJECT
+        
+2. Initialize the datmo project
+
+        $ datmo init
+
+3. Copy/save example files within project folder (if directory, copy the contents of the directory)
+
+        $ cp /path/to/SCRIPT_NAME.py .
+        $ cp /path/to/DIRECTORY/* .
+        
+4. Run 
+
+        $ python SCRIPT_NAME.py
 
 ### Jupyter Notebook
-1. `$ pip install datmo`
-2. Navigate to desired project folder
-3. `$ datmo init`
-4. Copy/save files within project folder
-5. Run `jupyter notebook`, navigate to `.ipynb` file
-6. Run cells in order
+1. Navigate to desired project folder or create a new one 
+
+        $ mkdir MY_DATMO_PROJECT
+        $ cd MY_DATMO_PROJECT
+        
+2. Initialize the datmo project
+
+        $ datmo init
+
+3. Copy/save example files within project folder (if directory, copy the contents of the directory)
+
+        $ cp /path/to/SCRIPT_NAME.ipynb .
+        $ cp /path/to/DIRECTORY/* .
+        
+4. Run 
+
+        $ jupyter notebook SCRIPT_NAME.ipynb
+
+5. Run cells in order
 
 
 ## Examples
 #### Creating a Snapshot
 * CLI
 	* TBD
 * Python
-	* [sklearn iris classifier](https://github.com/datmo/datmo/blob/master/examples/python/snapshot_create_iris_sklearn.py)
+	* [Snapshot create with sklearn iris classifier](https://github.com/datmo/datmo/blob/master/examples/python/snapshot_create_iris_sklearn.py)
 * Jupyter notebook
-	* [sklearn iris classifier](https://github.com/datmo/datmo/blob/master/examples/jupyter_notebook/snapshot_create_iris_sklearn.ipynb)
+	* [Snapshot create with sklearn iris classifier directory](https://github.com/datmo/datmo/blob/master/examples/jupyter_notebook/snapshot_create_iris_sklearn/)
+	    * [Snapshot create with sklearn iris classifier notebook](https://github.com/datmo/datmo/blob/master/examples/jupyter_notebook/snapshot_create_iris_sklearn/snapshot_create_iris_sklearn.ipynb)
 
 #### Running a containerized task (with option to create Snapshot)
 * CLI
 	* Run a training script (any language) (Coming soon)
 	* Run an inference script (any language) (Coming soon)
 * Python
-	* [Train a basic sklearn iris classifier](https://github.com/datmo/datmo/blob/master/examples/python/task_run_iris_sklearn/basic_task.py)
-	* [Run multiple tasks and snapshot the best result](https://github.com/datmo/datmo/blob/master/examples/python/task_run_iris_sklearn/basic_task.py)
+    * [Task run example with sklearn iris classifier directory](https://github.com/datmo/datmo/blob/master/examples/python/task_run_iris_sklearn)
+	    * [Train a basic sklearn iris classifier](https://github.com/datmo/datmo/blob/master/examples/python/task_run_iris_sklearn/basic_task.py)
+	    * [Run multiple tasks and snapshot the best result](https://github.com/datmo/datmo/blob/master/examples/python/task_run_iris_sklearn/task_compare.py)
     * Perform inference on saved model (iris sklearn) (Coming soon)
 * Jupyter notebook
     * TBD

examples/cli/snapshot_create_iris_sklearn.py

@@ -0,0 +1,21 @@
+from sklearn import datasets
+from sklearn.linear_model import LogisticRegression
+from sklearn.model_selection import train_test_split
+import datmo # extra line
+
+# Run `$ datmo init` in your terminal to instantiate a Datmo project
+
+config = { "solver": "newton-cg" } # extra line
+iris_dataset = datasets.load_iris()
+X, y = iris_dataset.data, iris_dataset.target
+X_train, X_test, y_train, y_test = train_test_split(X, y)
+model = LogisticRegression(**config).fit(X_train, y_train)
+
+train_acc = model.score(X_train, y_train)
+test_acc = model.score(X_test, y_test)
+
+print(train_acc)
+print(test_acc)
+
+stats = { "train_accuracy": train_acc, "test_accuracy": test_acc } # extra line
+datmo.snapshot.create(message="my first snapshot", config=config, stats=stats) # extra line

examples/cli/snapshot_create_iris_sklearn.sh

@@ -1,24 +1,14 @@
 #!/usr/bin/env bash
 
-# --- TEMPORARY ---
-# This script will create a datmo-enabled repository called `iris_sklearn`
+# This script will create a datmo-enabled repository in the current directory
 # run the initial script to create a snapshot and will list out the snapshots
 # by calling the `datmo snapshot ls` command
 
-# Create directory for the datmo-enabled repo to live
-mkdir iris_sklearn
-
-cd iris_sklearn
-
 # Create a datmo-enabled repo
 datmo init --name="iris data with sklearn models" --description="use iris data along with sklearn models"
 
-# Copy the relevant scripts into the repo
-cp ../iris_sklearn.py .
-cp ../iris_sklearn.ipynb .
-
 # Run the script to create a snapshot
-python iris_sklearn.py
+python snapshot_create_iris_sklearn.py
 
 # Run the datmo command to list all snapshots
 datmo snapshot ls

examples/jupyter_notebook/snapshot_create_iris_sklearn/Dockerfile

@@ -0,0 +1 @@
+FROM datmo/xgboost:cpu