Merge branch 'doc_updates'

docs/source/index.rst

@@ -26,6 +26,7 @@ See :doc:`the quickstart <quickstart>` to get started.
    :maxdepth: 1
 
    install
+   quickstart
    usage
    api
 

docs/source/install.rst

@@ -6,7 +6,7 @@ Easy
 
 Use ``pip`` or ``conda`` to install::
 
-   $ conda install knit
+   $ conda install knit -c blaze
    or
    $ pip install knit --upgrade
 

docs/source/quickstart.rst

@@ -1,16 +1,19 @@
 Quickstart
-----------
+==========
 
 Install
-~~~~~~~
+-------
 
 Use ``pip`` or ``conda`` to install::
 
    $ pip install knit --upgrade
-   $ conda install knit
+   $ conda install knit -c blaze
 
-Start Command
-~~~~~~~~~~~~~
+Commands
+--------
+
+Start
+~~~~~
 
 Instantiate ``knit`` with valid ResourceManager/Namenode IP/Ports and create a command string to run
 in all YARN containers
@@ -20,9 +23,10 @@ in all YARN containers
    >>> from knit import Knit
    >>> k = Knit(autodetect=True) # autodetect IP/Ports for YARN/HADOOP
    >>> cmd = 'date'
-   >>> app_id = k.start(cmd)
+   >>> k.start(cmd)
+   'application_1454900586318_0004'
 
-``start`` also take parameters: ``num_containers``, ``memory``, and ``virtual_cores``
+``start`` also takes parameters: ``num_containers``, ``memory``, ``virtual_cores``, and ``env``
 
 Status
 ~~~~~~
@@ -89,7 +93,7 @@ To stop an application from executing immediately, use the ``kill`` method:
 
 
 Python Applications
-~~~~~~~~~~~~~~~~~~~
+-------------------
 
 A simple Python based application:
 

docs/source/usage.rst

@@ -1,10 +1,15 @@
 Usage
 =====
 
+``Knit`` can be used in several novel ways.  Our primary concern is supporting easy deployment of
+distributed Python runtimes; though, we can also consider other languages (R, Julia, etc) should
+interest develop.  Below are a few novels ways we can currently use ``Knit``
 
 Python
 ~~~~~~
 
+The example below use Python found in the $PATH setting.  This is usually the system Python.
+
 .. code-block:: python
 
    >>> import knit

knit/core.py

@@ -18,7 +18,7 @@
 JAVA_APP = "io.continuum.knit.Client"
 
 
-class Knit(YARNAPI):
+class Knit(object):
     """
     Connection to HDFS/YARN
 
@@ -59,7 +59,8 @@ def __init__(self, nn="localhost", nn_port=9000,
             # validates IP/Port is correct
             self._hdfs_conf()
             self._yarn_conf()
-        super(Knit, self).__init__(rm, rm_port)
+
+        self.yarn_api = YARNAPI(rm, rm_port)
 
         self.java_lib_dir = os.path.join(os.path.dirname(__file__), "java_libs")
         self.KNIT_HOME = os.environ.get('KNIT_HOME') or self.java_lib_dir
@@ -236,3 +237,57 @@ def create_env(env_name, packages=None, conda_root=None, remove=False):
         path = c.create_env(env_name, packages=packages, remove=remove)
 
         return path
+
+    def logs(self, app_id, shell=False):
+        """
+        Collect logs from RM (if running)
+        With shell=True, collect logs from HDFS after job completion
+
+        Parameters
+        ----------
+        app_id: str
+             A yarn application ID string
+        shell: bool
+             Shell out to yarn CLI (default False)
+
+        Returns
+        -------
+        log: dictionary
+            logs from each container (when possible)
+        """
+        return self.yarn_api.logs(app_id, shell=shell)
+
+    def kill(self, app_id):
+        """
+        Method to kill a yarn application
+
+        Parameters
+        ----------
+        app_id: str
+            YARN application id
+
+        Returns
+        -------
+        bool:
+            True if successful, False otherwise.
+        """
+        return self.yarn_api.kill(app_id)
+
+    def status(self, app_id):
+        """ Get status of an application
+
+        Parameters
+        ----------
+        app_id: str
+             A yarn application ID string
+
+        Returns
+        -------
+        log: dictionary
+            status of application
+        """
+        return self.yarn_api.status(app_id)
+
+    @property
+    def apps(self):
+        return self.yarn_api.apps

knit/utils.py

@@ -10,7 +10,7 @@
 
 format = ('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logging.basicConfig(format=format, level=logging.INFO)
-
+logging.getLogger("requests").setLevel(logging.WARNING)
 
 def set_logging(level):
     logger = logging.getLogger('knit')

knit/yarn_api.py

@@ -3,6 +3,7 @@
 import os
 import requests
 import logging
+from functools import wraps
 from subprocess import Popen, PIPE
 
 from .utils import shell_out
@@ -11,6 +12,7 @@
 
 
 def check_app_id(func):
+    @wraps(func)
     def wrapper(*args, **kwargs):
         cls = args[0]
         app_id = args[1]