Support YARN endpoints protected by Kerberos/SPNEGO

.gitattributes

@@ -0,0 +1,12 @@
+# Set the default behavior to have all files normalized to Unix-style
+# line endings upon check-in.
+* text=auto
+ # Declare files that will always have CRLF line endings on checkout.
+*.bat text eol=crlf
+ # Denote all files that are truly binary and should not be modified.
+*.dll binary
+*.exp binary
+*.lib binary
+*.pdb binary
+*.exe binary
+

.gitignore

@@ -2,6 +2,27 @@
 *~
 \#*\#
 
+# Mac
+.DS_Store
+
+# Eclipse
+.classpath
+.project
+.settings/
+target/
+
+# Intellij
+.idea/
+.idea_modules/
+*.iml
+*.iws
+*.class
+*.log
+
+# Others
+.checkstyle
+.fbExcludeFilterFile
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -55,4 +76,3 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
-

.travis.yml

@@ -13,7 +13,8 @@ matrix:
 
 
 install:
-  - pip install tox coveralls
+  - pip install --upgrade pip tox coveralls
+  - pip freeze
 
 script:
   tox

README.rst

@@ -2,11 +2,7 @@
 hadoop-yarn-api-python-client
 =============================
 
-Python client for Hadoop® YARN API
-
-.. image:: https://coveralls.io/repos/toidi/hadoop-yarn-api-python-client/badge.png
-    :target: https://coveralls.io/r/toidi/hadoop-yarn-api-python-client
-    :alt: Test coverage
+Python client for Apache Hadoop® YARN API
 
 .. image:: https://img.shields.io/pypi/v/yarn-api-client.svg
     :target: https://pypi.python.org/pypi/yarn-api-client/
@@ -16,6 +12,14 @@ Python client for Hadoop® YARN API
     :target: https://travis-ci.org/toidi/hadoop-yarn-api-python-client
     :alt: Travis CI build status
 
+.. image:: http://readthedocs.org/projects/python-client-for-hadoop-yarn-api/badge/?version=latest
+    :target: https://python-client-for-hadoop-yarn-api.readthedocs.org/en/latest/?badge=latest
+    :alt: Latest documentation status
+
+.. image:: https://coveralls.io/repos/toidi/hadoop-yarn-api-python-client/badge.png
+    :target: https://coveralls.io/r/toidi/hadoop-yarn-api-python-client
+    :alt: Test coverage
+
 Package documentation: python-client-for-hadoop-yarn-api.readthedocs.org_
 
 REST API documentation: hadoop.apache.org_
@@ -30,6 +34,11 @@ From PyPI
 
     pip install yarn-api-client
 
+From Anaconda (conda forge)
+
+::
+
+    conda install -c conda-forge yarn-api-client
 
 From source code
 
@@ -67,9 +76,16 @@ Programmatic interface
 Changelog
 =========
 
-0.2.5 - Fixed History REST API
+0.3.0 Release
+    - Add support for YARN endpoints protected by Kerberos/SPNEGO
+    - Moved to `requests` package for REST API invocation
+    - Remove `http_con` property, as connections are now managed by `requests` package
+
+0.2.5 Release
+    - Fixed History REST API
 
-0.2.4 - Added compatibility with HA enabled Resource Manager
+0.2.4 Release
+    - Added compatibility with HA enabled Resource Manager
 
 .. _python-client-for-hadoop-yarn-api.readthedocs.org: http://python-client-for-hadoop-yarn-api.readthedocs.org/en/latest/
 .. _hadoop.apache.org: http://hadoop.apache.org/docs/stable/hadoop-yarn/hadoop-yarn-site/WebServicesIntro.html

itests/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+try:
+    from unittest2 import TestCase
+except ImportError:
+    from unittest import TestCase

itests/integration_test_resource_manager.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+import os
+
+from pprint import pprint
+from unittest import TestCase
+from yarn_api_client.resource_manager import ResourceManager
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+
+class ResourceManagerTestCase(TestCase):
+    """
+    Integration test that, given a provided YARN ENDPOINT,
+    execute some real scenario test against that server.
+
+    Note that, if no YARN ENDPOINT is provided, the tests
+    are ignored.
+    """
+    @classmethod
+    def setUpClass(self):
+        self.configured = False
+        if os.getenv('YARN_ENDPOINT'):
+            yarn_endpoint = os.getenv('YARN_ENDPOINT')
+            yarn_endpoint_uri = urlparse(yarn_endpoint)
+
+            if yarn_endpoint_uri.hostname and yarn_endpoint_uri.port:
+                self.configured = True
+                self.resourceManager = ResourceManager(yarn_endpoint_uri.hostname, yarn_endpoint_uri.port)
+
+    def test_cluster_information(self):
+        if self.configured:
+            info = self.resourceManager.cluster_information()
+            pprint(info.data)
+            self.assertEqual(info.data['clusterInfo']['state'], 'STARTED')
+
+    def test_cluster_metrics(self):
+        if self.configured:
+            metrics = self.resourceManager.cluster_metrics()
+            pprint(metrics.data)
+            self.assertGreater(metrics.data['clusterMetrics']['activeNodes'], 0)
+            self.assertIsNotNone(metrics.data['clusterMetrics']['totalNodes'])
+
+    def test_cluster_scheduler(self):
+        if self.configured:
+            scheduler = self.resourceManager.cluster_scheduler()
+            pprint(scheduler.data)
+            self.assertIsNotNone(scheduler.data['scheduler']['schedulerInfo'])
+
+    def test_cluster_applications(self):
+        if self.configured:
+            apps = self.resourceManager.cluster_applications()
+            pprint(apps.data)
+            self.assertIsNotNone(apps.data['apps'])
+
+    def test_cluster_application_state(self):
+        if self.configured:
+            apps = self.resourceManager.cluster_applications()
+            appid = apps.data['apps']['app'][0]['id']
+            print(appid)
+            response = self.resourceManager.cluster_application_state(appid)
+            pprint(response.data)
+            pprint(response.data['state'])
+            self.assertIsNotNone(apps.data['apps'])
+
+    def test_cluster_application_statistics(self):
+        if self.configured:
+            appstats = self.resourceManager.cluster_application_statistics()
+            pprint(appstats.data)
+            self.assertIsNotNone(appstats.data['appStatInfo'])
+
+    def test_cluster_nodes(self):
+        if self.configured:
+            nodes = self.resourceManager.cluster_nodes()
+            pprint(nodes.data)
+            self.assertIsNotNone(nodes.data['nodes'])
+
+            running_nodes = self.resourceManager.cluster_nodes(state='RUNNING', healthy='true')
+            pprint(running_nodes.data)
+            self.assertIsNotNone(nodes.data['nodes'])

setup.cfg

@@ -0,0 +1,6 @@
+[bdist_wheel]
+universal=1
+
+[metadata]
+description-file=README.rst
+license_file = LICENSE

setup.py

@@ -19,17 +19,17 @@ def find_version(*file_paths):
         return version_match.group(1)
     raise RuntimeError("Unable to find version string.")
 
-
-install_requires = []
-
 setup(
     name = 'yarn-api-client',
     version = find_version('yarn_api_client', '__init__.py'),
     description='Python client for Hadoop® YARN API',
     long_description=read('README.rst'),
-    packages = find_packages(exclude=['tests']),
+    packages = find_packages(exclude=['tests','itests']),
 
-    install_requires = install_requires,
+    install_requires = [
+        'requests>=2.7,<3.0',
+        'requests-kerberos',
+    ],
     entry_points = {
         'console_scripts': [
             'yarn_client = yarn_api_client.main:main',

tests/test_base.py

@@ -4,56 +4,66 @@
 except ImportError:
     from http.client import OK
 
-from mock import patch
-from tests import TestCase
+import json
+import requests
+import requests_mock
 
+from tests import TestCase
 from yarn_api_client import base
 from yarn_api_client.errors import APIError, ConfigurationError
 
 
 class BaseYarnAPITestCase(TestCase):
-    def test_request(self):
-        client = self.get_client()
-        with patch('yarn_api_client.base.HTTPConnection') as http_conn_mock:
-            with patch('yarn_api_client.base.json'):
-                http_conn_mock().getresponse().status = OK
+    @staticmethod
+    def success_response():
+        return {
+            'status':'success'
+        }
 
-                client.request('/ololo', foo='bar')
+    def test_valid_request(self):
+        with requests_mock.mock() as requests_get_mock:
+            requests_get_mock.get('/ololo', text=json.dumps(BaseYarnAPITestCase.success_response()))
 
-                http_conn_mock().request.assert_called_with('GET', '/ololo?foo=bar')
+            client = self.get_client()
+            response = client.request('/ololo', foo='bar')
 
-                http_conn_mock.reset_mock()
-                client.request('/ololo')
+            assert requests_get_mock.called
+            self.assertIn(response.data['status'], 'success')
+
+
+    def test_valid_request_with_parameters(self):
+        with requests_mock.mock() as requests_get_mock:
+            requests_get_mock.get('/ololo?foo=bar', text=json.dumps(BaseYarnAPITestCase.success_response()))
 
-                http_conn_mock()
+            client = self.get_client()
+            response = client.request('/ololo', foo='bar')
 
-                http_conn_mock().request.assert_called_with('GET', '/ololo')
+            assert requests_get_mock.called
+            self.assertIn(response.data['status'], 'success')
 
     def test_bad_request(self):
-        client = self.get_client()
-        with patch('yarn_api_client.base.HTTPConnection') as http_conn_mock:
-            http_conn_mock().getresponse().status = 404
+        with requests_mock.mock() as requests_get_mock:
+            requests_get_mock.get('/ololo', status_code=404)
 
+            client = self.get_client()
             with self.assertRaises(APIError):
                 client.request('/ololo')
-
-    def test_http_configuration(self):
-        client = self.get_client()
-        client.address = None
-        client.port = 80
 
-        with self.assertRaises(ConfigurationError):
-            conn = client.http_conn
+    def test_http_configuration(self):
+        with requests_mock.mock() as requests_get_mock:
+            requests_get_mock.get('/ololo', text=json.dumps(BaseYarnAPITestCase.success_response()))
 
-        client.address = 'localhost'
-        client.port = None
+            client = self.get_client()
+            client.address = None
+            client.port = 80
 
-        with self.assertRaises(ConfigurationError):
-            conn = client.http_conn
+            with self.assertRaises(ConfigurationError):
+                client.request('/ololo')
 
     def get_client(self):
         client = base.BaseYarnAPI()
         client.address = 'example.com'
         client.port = 80
         client.timeout = 0
+        client.kerberos_enabled = False
         return client

tox.ini

@@ -1,15 +1,11 @@
 [tox]
-envlist = py26,py27,py34,py35
+envlist = py27,py35,py36
 
 [testenv]
 deps =
-     coverage
-     mock
+    coverage
+    mock
+    requests
+    requests-kerberos
+    requests_mock
 commands = coverage run --source=yarn_api_client setup.py test
-
-[testenv:py26]
-deps =
-     argparse
-     coverage
-     mock
-     unittest2

yarn_api_client/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = '0.2.5'
+__version__ = '0.3.0'
 __all__ = ['ApplicationMaster', 'HistoryServer', 'NodeManager',
            'ResourceManager']
 

yarn_api_client/application_master.py

@@ -18,9 +18,10 @@ class ApplicationMaster(BaseYarnAPI):
     :param str address: Proxy HTTP address
     :param int port: Proxy HTTP port
     :param int timeout: API connection timeout in seconds
+    :param boolean kerberos_enabled: Flag identifying is Kerberos Security has been enabled for YARN
     """
-    def __init__(self, address=None, port=8088, timeout=30):
-        self.address, self.port, self.timeout = address, port, timeout
+    def __init__(self, address=None, port=8088, timeout=30, kerberos_enabled=False):
+        self.address, self.port, self.timeout, self.kerberos_enabled = address, port, timeout, kerberos_enabled
         if address is None:
             self.logger.debug('Get configuration from hadoop conf dir')
             address, port = get_webproxy_host_port()