diff --git a/.travis.yml b/.travis.yml index 428b02d95..edfd7efcc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ before_install: env: - SPLUNK_VERSION=7.0-sdk - SPLUNK_VERSION=7.2-sdk + - SPLUNK_VERSION=8.0-sdk language: python diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f9a9c864..083a9595d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Splunk SDK for Python Changelog +## Version 1.6.12 + +### New features and APIs +* Added Bearer token support using Splunk Token in v7.3 +* Made modinput text consistent + +### Bug fixes +* Changed permissions from 755 to 644 for python files to pass appinspect checks +* Removed version check on ssl verify toggle + ## Version 1.6.11 ### Bug Fix diff --git a/README.md b/README.md index d4b819cde..4cad66efa 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Software Development Kit for Python -#### Version 1.6.11 +#### Version 1.6.12 The Splunk Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using Splunk. @@ -17,7 +17,7 @@ monitoring of IT machine data, security, compliance and a wide variety of other scenarios that share a requirement to efficiently index, search, analyze and generate real-time notifications from large volumes of time series data. -The Splunk developer platform enables developers to take advantage of the same +The Splunk developer platform enables ^Fdevelopers to take advantage of the same technology used by the Splunk product to build exciting new applications that are enabled by Splunk's unique capabilities. diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py index b84d5bbb4..2cf51a4c2 100755 --- a/examples/searchcommands_app/setup.py +++ b/examples/searchcommands_app/setup.py @@ -439,7 +439,7 @@ def run(self): setup( description='Custom Search Command examples', name=os.path.basename(project_dir), - version='1.6.11', + version='1.6.12', author='Splunk, Inc.', author_email='devinfo@splunk.com', url='http://github.com/splunk/splunk-sdk-python', diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 7b3767120..929a63172 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 11) +__version_info__ = (1, 6, 12) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index bfccf6f3b..b0ed20e1b 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -450,6 +450,8 @@ class Context(object): :type username: ``string`` :param password: The password for the Splunk account. :type password: ``string`` + :param splunkToken: Splunk authentication token + :type splunkToken: ``string`` :param headers: List of extra HTTP headers to send (optional). :type headers: ``list`` of 2-tuples. :param handler: The HTTP request handler (optional). @@ -481,6 +483,7 @@ def __init__(self, handler=None, **kwargs): self.username = kwargs.get("username", "") self.password = kwargs.get("password", "") self.basic = kwargs.get("basic", False) + self.bearerToken = kwargs.get("splunkToken", "") self.autologin = kwargs.get("autologin", False) self.additional_headers = kwargs.get("headers", []) @@ -521,6 +524,9 @@ def _auth_headers(self): elif self.basic and (self.username and self.password): token = 'Basic %s' % b64encode(("%s:%s" % (self.username, self.password)).encode('utf-8')).decode('ascii') return [("Authorization", token)] + elif self.bearerToken: + token = 'Bearer %s' % self.bearerToken + return [("Authorization", token)] elif self.token is _NoAuthenticationToken: return [] else: @@ -863,6 +869,10 @@ def login(self): # as credentials were passed in. return + if self.bearerToken: + # Bearer auth mode requested, so this method is a nop as long + # as authentication token was passed in. + return # Only try to get a token and updated cookie if username & password are specified try: response = self.http.post( @@ -1357,8 +1367,7 @@ def connect(scheme, host, port): if key_file is not None: kwargs['key_file'] = key_file if cert_file is not None: kwargs['cert_file'] = cert_file - # If running Python 2.7.9+, disable SSL certificate validation - if (sys.version_info >= (2,7,9) and key_file is None and cert_file is None) and not verify: + if not verify: kwargs['context'] = ssl._create_unverified_context() return six.moves.http_client.HTTPSConnection(host, port, **kwargs) raise ValueError("unsupported scheme: %s" % scheme) @@ -1369,7 +1378,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.11", + "User-Agent": "splunk-sdk-python/1.6.12", "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/modularinput/__init__.py b/splunklib/modularinput/__init__.py old mode 100755 new mode 100644 diff --git a/splunklib/modularinput/argument.py b/splunklib/modularinput/argument.py old mode 100755 new mode 100644 diff --git a/splunklib/modularinput/event.py b/splunklib/modularinput/event.py old mode 100755 new mode 100644 index fdf19fa13..9cd6cf3ae --- a/splunklib/modularinput/event.py +++ b/splunklib/modularinput/event.py @@ -13,6 +13,9 @@ # under the License. from __future__ import absolute_import +from io import TextIOBase +from splunklib.six import ensure_text + try: import xml.etree.cElementTree as ET except ImportError as ie: @@ -104,5 +107,8 @@ def write_to(self, stream): if self.done: ET.SubElement(event, "done") - stream.write(ET.tostring(event)) + if isinstance(stream, TextIOBase): + stream.write(ensure_text(ET.tostring(event))) + else: + stream.write(ET.tostring(event)) stream.flush() \ No newline at end of file diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py old mode 100755 new mode 100644 index fb96c9149..4d0b21f69 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -15,6 +15,8 @@ from __future__ import absolute_import import sys +from io import TextIOWrapper, TextIOBase +from splunklib.six import ensure_text from .event import ET try: @@ -24,7 +26,6 @@ class EventWriter(object): """``EventWriter`` writes events and error messages to Splunk from a modular input. - Its two important methods are ``writeEvent``, which takes an ``Event`` object, and ``log``, which takes a severity and an error message. """ @@ -42,8 +43,15 @@ def __init__(self, output = sys.stdout, error = sys.stderr): :param output: Where to write the output; defaults to sys.stdout. :param error: Where to write any errors; defaults to sys.stderr. """ - self._out = output - self._err = error + if isinstance(output, TextIOBase): + self._out = output + else: + self._out = TextIOWrapper(output) + + if isinstance(error, TextIOBase): + self._err = error + else: + self._err = TextIOWrapper(error) # has the opening tag been written yet? self.header_written = False @@ -55,7 +63,7 @@ def write_event(self, event): """ if not self.header_written: - self._out.write(b"") + self._out.write(ensure_text("")) self.header_written = True event.write_to(self._out) @@ -63,12 +71,10 @@ def write_event(self, event): def log(self, severity, message): """Logs messages about the state of this modular input to Splunk. These messages will show up in Splunk's internal logs. - :param severity: ``string``, severity of message, see severities defined as class constants. :param message: ``string``, message to log. """ - - self._err.write(("%s %s\n" % (severity, message)).encode('utf-8')) + self._err.write(ensure_text("%s %s\n" % (severity, message))) self._err.flush() def write_xml_document(self, document): @@ -77,9 +83,11 @@ def write_xml_document(self, document): :param document: An ``ElementTree`` object. """ - self._out.write(ET.tostring(document)) + data = ET.tostring(document) + self._out.write(ensure_text(data)) self._out.flush() def close(self): """Write the closing tag to make this XML well formed.""" - self._out.write(b"") + self._out.write(ensure_text("")) + self._out.flush() diff --git a/splunklib/modularinput/input_definition.py b/splunklib/modularinput/input_definition.py old mode 100755 new mode 100644 diff --git a/splunklib/modularinput/scheme.py b/splunklib/modularinput/scheme.py old mode 100755 new mode 100644 diff --git a/splunklib/modularinput/script.py b/splunklib/modularinput/script.py old mode 100755 new mode 100644 diff --git a/splunklib/modularinput/utils.py b/splunklib/modularinput/utils.py old mode 100755 new mode 100644 diff --git a/splunklib/modularinput/validation_definition.py b/splunklib/modularinput/validation_definition.py old mode 100755 new mode 100644 diff --git a/splunklib/six.py b/splunklib/six.py index 190c0239c..5fe9f8e14 100644 --- a/splunklib/six.py +++ b/splunklib/six.py @@ -1,6 +1,4 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2015 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Utilities for writing code that runs on Python 2 and 3""" + from __future__ import absolute_import import functools @@ -29,7 +29,7 @@ import types __author__ = "Benjamin Peterson " -__version__ = "1.10.0" +__version__ = "1.14.0" # Useful for very coarse version differentiation. @@ -241,6 +241,7 @@ class _MovedItems(_LazyModule): MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), MovedAttribute("reduce", "__builtin__", "functools"), @@ -254,18 +255,21 @@ class _MovedItems(_LazyModule): MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -337,10 +341,12 @@ class Module_six_moves_urllib_parse(_LazyModule): MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), MovedAttribute("uses_params", "urlparse", "urllib.parse"), @@ -416,6 +422,8 @@ class Module_six_moves_urllib_request(_LazyModule): MovedAttribute("URLopener", "urllib", "urllib.request"), MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) @@ -631,13 +639,16 @@ def u(s): import io StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): return s @@ -659,6 +670,7 @@ def indexbytes(buf, i): _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -675,15 +687,23 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None else: def exec_(_code_, _globs_=None, _locs_=None): @@ -699,19 +719,19 @@ def exec_(_code_, _globs_=None, _locs_=None): exec("""exec _code_ in _globs_, _locs_""") exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb + try: + raise tp, value, tb + finally: + tb = None """) -if sys.version_info[:2] == (3, 2): +if sys.version_info[:2] > (3,): exec_("""def raise_from(value, from_value): - if from_value is None: - raise value - raise value from from_value -""") -elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): - raise value from from_value + try: + raise value from from_value + finally: + value = None """) else: def raise_from(value, from_value): @@ -786,13 +806,33 @@ def print_(*args, **kwargs): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - return wrapper + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + else: wraps = functools.wraps @@ -802,10 +842,22 @@ def with_metaclass(meta, *bases): # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): + class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) return type.__new__(metaclass, 'temporary_class', (), {}) @@ -821,13 +873,73 @@ def wrapper(cls): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method diff --git a/tests/modularinput/test_event.py b/tests/modularinput/test_event.py index b774ef963..58be3d4ea 100644 --- a/tests/modularinput/test_event.py +++ b/tests/modularinput/test_event.py @@ -18,7 +18,7 @@ from tests.modularinput.modularinput_testlib import unittest, xml_compare, data_open from splunklib.modularinput.event import Event, ET from splunklib.modularinput.event_writer import EventWriter -from io import BytesIO +from io import BytesIO, TextIOWrapper try: from splunklib.six.moves import cStringIO as StringIO diff --git a/tests/modularinput/test_script.py b/tests/modularinput/test_script.py index f8218b298..c334edb7d 100644 --- a/tests/modularinput/test_script.py +++ b/tests/modularinput/test_script.py @@ -21,12 +21,7 @@ from splunklib.modularinput.script import Script from splunklib.modularinput.scheme import Scheme -try: - from splunklib.six.moves import cStringIO as StringIO -except ImportError: - from splunklib.six import StringIO - -from io import BytesIO +from io import StringIO, BytesIO try: import xml.etree.cElementTree as ET diff --git a/tests/test_examples.py b/tests/test_examples.py index 059d54645..f35905fc2 100755 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -248,6 +248,8 @@ def test_submit(self): def test_upload(self): # Note: test must run on machine where splunkd runs, # or a failure is expected + if "SPLUNK_HOME" not in os.environ: + self.skipTest("SPLUNK_HOME is not set, skipping") file_to_upload = os.path.expandvars(os.environ.get("INPUT_EXAMPLE_UPLOAD", "./upload.py")) self.check_commands( "upload.py --help", diff --git a/tests/test_index.py b/tests/test_index.py index b2fa9d0d9..aa1ce7531 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -40,9 +40,9 @@ def tearDown(self): # someone cares to go clean them up. Unique naming prevents # clashes, though. if self.service.splunk_version >= (5,): - if self.index_name in self.service.indexes and "TRAVIS" in os.environ: + if self.index_name in self.service.indexes: self.service.indexes.delete(self.index_name) - self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) + self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) else: logging.warning("test_index.py:TestDeleteIndex: Skipped: cannot " "delete indexes via the REST API in Splunk 4.x")