From 85807ef1eb3b8bfcc3aca7c9b7f10e2a92492ae4 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Fri, 15 Mar 2024 12:13:58 +0530 Subject: [PATCH] reverted line separators - line separators were mistakenly updated, hence reverting it back. --- splunklib/binding.py | 2994 +++---- splunklib/client.py | 7816 +++++++++---------- splunklib/searchcommands/search_command.py | 2286 +++--- tests/searchcommands/chunked_data_stream.py | 200 +- tests/searchcommands/test_internals_v1.py | 686 +- tests/test_binding.py | 1950 ++--- tests/testlib.py | 522 +- 7 files changed, 8227 insertions(+), 8227 deletions(-) mode change 100644 => 100755 tests/searchcommands/test_internals_v1.py mode change 100644 => 100755 tests/test_binding.py diff --git a/splunklib/binding.py b/splunklib/binding.py index 7437fc2b..958be96e 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -1,1497 +1,1497 @@ -# Copyright © 2011-2024 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""The **splunklib.binding** module provides a low-level binding interface to the -`Splunk REST API `_. - -This module handles the wire details of calling the REST API, such as -authentication tokens, prefix paths, URL encoding, and so on. Actual path -segments, ``GET`` and ``POST`` arguments, and the parsing of responses is left -to the user. - -If you want a friendlier interface to the Splunk REST API, use the -:mod:`splunklib.client` module. -""" - -import io -import json -import logging -import socket -import ssl -import time -from base64 import b64encode -from contextlib import contextmanager -from datetime import datetime -from functools import wraps -from io import BytesIO -from urllib import parse -from http import client -from http.cookies import SimpleCookie -from xml.etree.ElementTree import XML, ParseError -from splunklib.data import record -from splunklib import __version__ - - -logger = logging.getLogger(__name__) - -__all__ = [ - "AuthenticationError", - "connect", - "Context", - "handler", - "HTTPError", - "UrlEncoded", - "_encode", - "_make_cookie_header", - "_NoAuthenticationToken", - "namespace" -] - -SENSITIVE_KEYS = ['Authorization', 'Cookie', 'action.email.auth_password', 'auth', 'auth_password', 'clear_password', 'clientId', - 'crc-salt', 'encr_password', 'oldpassword', 'passAuth', 'password', 'session', 'suppressionKey', - 'token'] - -# If you change these, update the docstring -# on _authority as well. -DEFAULT_HOST = "localhost" -DEFAULT_PORT = "8089" -DEFAULT_SCHEME = "https" - - -def _log_duration(f): - @wraps(f) - def new_f(*args, **kwargs): - start_time = datetime.now() - val = f(*args, **kwargs) - end_time = datetime.now() - logger.debug("Operation took %s", end_time - start_time) - return val - - return new_f - - -def mask_sensitive_data(data): - ''' - Masked sensitive fields data for logging purpose - ''' - if not isinstance(data, dict): - try: - data = json.loads(data) - except Exception as ex: - return data - - # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type - if not isinstance(data, dict): - return data - mdata = {} - for k, v in data.items(): - if k in SENSITIVE_KEYS: - mdata[k] = "******" - else: - mdata[k] = mask_sensitive_data(v) - return mdata - - -def _parse_cookies(cookie_str, dictionary): - """Tries to parse any key-value pairs of cookies in a string, - then updates the the dictionary with any key-value pairs found. - - **Example**:: - - dictionary = {} - _parse_cookies('my=value', dictionary) - # Now the following is True - dictionary['my'] == 'value' - - :param cookie_str: A string containing "key=value" pairs from an HTTP "Set-Cookie" header. - :type cookie_str: ``str`` - :param dictionary: A dictionary to update with any found key-value pairs. - :type dictionary: ``dict`` - """ - parsed_cookie = SimpleCookie(cookie_str) - for cookie in parsed_cookie.values(): - dictionary[cookie.key] = cookie.coded_value - - -def _make_cookie_header(cookies): - """ - Takes a list of 2-tuples of key-value pairs of - cookies, and returns a valid HTTP ``Cookie`` - header. - - **Example**:: - - header = _make_cookie_header([("key", "value"), ("key_2", "value_2")]) - # Now the following is True - header == "key=value; key_2=value_2" - - :param cookies: A list of 2-tuples of cookie key-value pairs. - :type cookies: ``list`` of 2-tuples - :return: ``str` An HTTP header cookie string. - :rtype: ``str`` - """ - return "; ".join(f"{key}={value}" for key, value in cookies) - - -# Singleton values to eschew None -class _NoAuthenticationToken: - """The value stored in a :class:`Context` or :class:`splunklib.client.Service` - class that is not logged in. - - If a ``Context`` or ``Service`` object is created without an authentication - token, and there has not yet been a call to the ``login`` method, the token - field of the ``Context`` or ``Service`` object is set to - ``_NoAuthenticationToken``. - - Likewise, after a ``Context`` or ``Service`` object has been logged out, the - token is set to this value again. - """ - - -class UrlEncoded(str): - """This class marks URL-encoded strings. - It should be considered an SDK-private implementation detail. - - Manually tracking whether strings are URL encoded can be difficult. Avoid - calling ``urllib.quote`` to replace special characters with escapes. When - you receive a URL-encoded string, *do* use ``urllib.unquote`` to replace - escapes with single characters. Then, wrap any string you want to use as a - URL in ``UrlEncoded``. Note that because the ``UrlEncoded`` class is - idempotent, making multiple calls to it is OK. - - ``UrlEncoded`` objects are identical to ``str`` objects (including being - equal if their contents are equal) except when passed to ``UrlEncoded`` - again. - - ``UrlEncoded`` removes the ``str`` type support for interpolating values - with ``%`` (doing that raises a ``TypeError``). There is no reliable way to - encode values this way, so instead, interpolate into a string, quoting by - hand, and call ``UrlEncode`` with ``skip_encode=True``. - - **Example**:: - - import urllib - UrlEncoded(f'{scheme}://{urllib.quote(host)}', skip_encode=True) - - If you append ``str`` strings and ``UrlEncoded`` strings, the result is also - URL encoded. - - **Example**:: - - UrlEncoded('ab c') + 'de f' == UrlEncoded('ab cde f') - 'ab c' + UrlEncoded('de f') == UrlEncoded('ab cde f') - """ - - def __new__(self, val='', skip_encode=False, encode_slash=False): - if isinstance(val, UrlEncoded): - # Don't urllib.quote something already URL encoded. - return val - if skip_encode: - return str.__new__(self, val) - if encode_slash: - return str.__new__(self, parse.quote_plus(val)) - # When subclassing str, just call str.__new__ method - # with your class and the value you want to have in the - # new string. - return str.__new__(self, parse.quote(val)) - - def __add__(self, other): - """self + other - - If *other* is not a ``UrlEncoded``, URL encode it before - adding it. - """ - if isinstance(other, UrlEncoded): - return UrlEncoded(str.__add__(self, other), skip_encode=True) - - return UrlEncoded(str.__add__(self, parse.quote(other)), skip_encode=True) - - def __radd__(self, other): - """other + self - - If *other* is not a ``UrlEncoded``, URL _encode it before - adding it. - """ - if isinstance(other, UrlEncoded): - return UrlEncoded(str.__radd__(self, other), skip_encode=True) - - return UrlEncoded(str.__add__(parse.quote(other), self), skip_encode=True) - - def __mod__(self, fields): - """Interpolation into ``UrlEncoded``s is disabled. - - If you try to write ``UrlEncoded("%s") % "abc", will get a - ``TypeError``. - """ - raise TypeError("Cannot interpolate into a UrlEncoded object.") - - def __repr__(self): - return f"UrlEncoded({repr(parse.unquote(str(self)))})" - - -@contextmanager -def _handle_auth_error(msg): - """Handle re-raising HTTP authentication errors as something clearer. - - If an ``HTTPError`` is raised with status 401 (access denied) in - the body of this context manager, re-raise it as an - ``AuthenticationError`` instead, with *msg* as its message. - - This function adds no round trips to the server. - - :param msg: The message to be raised in ``AuthenticationError``. - :type msg: ``str`` - - **Example**:: - - with _handle_auth_error("Your login failed."): - ... # make an HTTP request - """ - try: - yield - except HTTPError as he: - if he.status == 401: - raise AuthenticationError(msg, he) - else: - raise - - -def _authentication(request_fun): - """Decorator to handle autologin and authentication errors. - - *request_fun* is a function taking no arguments that needs to - be run with this ``Context`` logged into Splunk. - - ``_authentication``'s behavior depends on whether the - ``autologin`` field of ``Context`` is set to ``True`` or - ``False``. If it's ``False``, then ``_authentication`` - aborts if the ``Context`` is not logged in, and raises an - ``AuthenticationError`` if an ``HTTPError`` of status 401 is - raised in *request_fun*. If it's ``True``, then - ``_authentication`` will try at all sensible places to - log in before issuing the request. - - If ``autologin`` is ``False``, ``_authentication`` makes - one roundtrip to the server if the ``Context`` is logged in, - or zero if it is not. If ``autologin`` is ``True``, it's less - deterministic, and may make at most three roundtrips (though - that would be a truly pathological case). - - :param request_fun: A function of no arguments encapsulating - the request to make to the server. - - **Example**:: - - import splunklib.binding as binding - c = binding.connect(..., autologin=True) - c.logout() - def f(): - c.get("/services") - return 42 - print(_authentication(f)) - """ - - @wraps(request_fun) - def wrapper(self, *args, **kwargs): - if self.token is _NoAuthenticationToken and not self.has_cookies(): - # Not yet logged in. - if self.autologin and self.username and self.password: - # This will throw an uncaught - # AuthenticationError if it fails. - self.login() - else: - # Try the request anyway without authentication. - # Most requests will fail. Some will succeed, such as - # 'GET server/info'. - with _handle_auth_error("Request aborted: not logged in."): - return request_fun(self, *args, **kwargs) - try: - # Issue the request - return request_fun(self, *args, **kwargs) - except HTTPError as he: - if he.status == 401 and self.autologin: - # Authentication failed. Try logging in, and then - # rerunning the request. If either step fails, throw - # an AuthenticationError and give up. - with _handle_auth_error("Autologin failed."): - self.login() - with _handle_auth_error("Authentication Failed! If session token is used, it seems to have been expired."): - return request_fun(self, *args, **kwargs) - elif he.status == 401 and not self.autologin: - raise AuthenticationError( - "Request failed: Session is not logged in.", he) - else: - raise - - return wrapper - - -def _authority(scheme=DEFAULT_SCHEME, host=DEFAULT_HOST, port=DEFAULT_PORT): - """Construct a URL authority from the given *scheme*, *host*, and *port*. - - Named in accordance with RFC2396_, which defines URLs as:: - - ://? - - .. _RFC2396: http://www.ietf.org/rfc/rfc2396.txt - - So ``https://localhost:8000/a/b/b?boris=hilda`` would be parsed as:: - - scheme := https - authority := localhost:8000 - path := /a/b/c - query := boris=hilda - - :param scheme: URL scheme (the default is "https") - :type scheme: "http" or "https" - :param host: The host name (the default is "localhost") - :type host: string - :param port: The port number (the default is 8089) - :type port: integer - :return: The URL authority. - :rtype: UrlEncoded (subclass of ``str``) - - **Example**:: - - _authority() == "https://localhost:8089" - - _authority(host="splunk.utopia.net") == "https://splunk.utopia.net:8089" - - _authority(host="2001:0db8:85a3:0000:0000:8a2e:0370:7334") == \ - "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089" - - _authority(scheme="http", host="splunk.utopia.net", port="471") == \ - "http://splunk.utopia.net:471" - - """ - # check if host is an IPv6 address and not enclosed in [ ] - if ':' in host and not (host.startswith('[') and host.endswith(']')): - # IPv6 addresses must be enclosed in [ ] in order to be well - # formed. - host = '[' + host + ']' - return UrlEncoded(f"{scheme}://{host}:{port}", skip_encode=True) - - -# kwargs: sharing, owner, app -def namespace(sharing=None, owner=None, app=None, **kwargs): - """This function constructs a Splunk namespace. - - Every Splunk resource belongs to a namespace. The namespace is specified by - the pair of values ``owner`` and ``app`` and is governed by a ``sharing`` mode. - The possible values for ``sharing`` are: "user", "app", "global" and "system", - which map to the following combinations of ``owner`` and ``app`` values: - - "user" => {owner}, {app} - - "app" => nobody, {app} - - "global" => nobody, {app} - - "system" => nobody, system - - "nobody" is a special user name that basically means no user, and "system" - is the name reserved for system resources. - - "-" is a wildcard that can be used for both ``owner`` and ``app`` values and - refers to all users and all apps, respectively. - - In general, when you specify a namespace you can specify any combination of - these three values and the library will reconcile the triple, overriding the - provided values as appropriate. - - Finally, if no namespacing is specified the library will make use of the - ``/services`` branch of the REST API, which provides a namespaced view of - Splunk resources equivelent to using ``owner={currentUser}`` and - ``app={defaultApp}``. - - The ``namespace`` function returns a representation of the namespace from - reconciling the values you provide. It ignores any keyword arguments other - than ``owner``, ``app``, and ``sharing``, so you can provide ``dicts`` of - configuration information without first having to extract individual keys. - - :param sharing: The sharing mode (the default is "user"). - :type sharing: "system", "global", "app", or "user" - :param owner: The owner context (the default is "None"). - :type owner: ``string`` - :param app: The app context (the default is "None"). - :type app: ``string`` - :returns: A :class:`splunklib.data.Record` containing the reconciled - namespace. - - **Example**:: - - import splunklib.binding as binding - n = binding.namespace(sharing="user", owner="boris", app="search") - n = binding.namespace(sharing="global", app="search") - """ - if sharing in ["system"]: - return record({'sharing': sharing, 'owner': "nobody", 'app': "system"}) - if sharing in ["global", "app"]: - return record({'sharing': sharing, 'owner': "nobody", 'app': app}) - if sharing in ["user", None]: - return record({'sharing': sharing, 'owner': owner, 'app': app}) - raise ValueError("Invalid value for argument: 'sharing'") - - -class Context: - """This class represents a context that encapsulates a splunkd connection. - - The ``Context`` class encapsulates the details of HTTP requests, - authentication, a default namespace, and URL prefixes to simplify access to - the REST API. - - After creating a ``Context`` object, you must call its :meth:`login` - method before you can issue requests to splunkd. Or, use the :func:`connect` - function to create an already-authenticated ``Context`` object. You can - provide a session token explicitly (the same token can be shared by multiple - ``Context`` objects) to provide authentication. - - :param host: The host name (the default is "localhost"). - :type host: ``string`` - :param port: The port number (the default is 8089). - :type port: ``integer`` - :param scheme: The scheme for accessing the service (the default is "https"). - :type scheme: "https" or "http" - :param verify: Enable (True) or disable (False) SSL verification for https connections. - :type verify: ``Boolean`` - :param sharing: The sharing mode for the namespace (the default is "user"). - :type sharing: "global", "system", "app", or "user" - :param owner: The owner context of the namespace (optional, the default is "None"). - :type owner: ``string`` - :param app: The app context of the namespace (optional, the default is "None"). - :type app: ``string`` - :param token: A session token. When provided, you don't need to call :meth:`login`. - :type token: ``string`` - :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. - This parameter is only supported for Splunk 6.2+. - :type cookie: ``string`` - :param username: The Splunk account username, which is used to - authenticate the Splunk instance. - :type username: ``string`` - :param password: The password for the Splunk account. - :type password: ``string`` - :param splunkToken: Splunk authentication token - :type splunkToken: ``string`` - :param headers: List of extra HTTP headers to send (optional). - :type headers: ``list`` of 2-tuples. - :param retires: Number of retries for each HTTP connection (optional, the default is 0). - NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER AND BLOCK THE - CURRENT THREAD WHILE RETRYING. - :type retries: ``int`` - :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). - :type retryDelay: ``int`` (in seconds) - :param handler: The HTTP request handler (optional). - :returns: A ``Context`` instance. - - **Example**:: - - import splunklib.binding as binding - c = binding.Context(username="boris", password="natasha", ...) - c.login() - # Or equivalently - c = binding.connect(username="boris", password="natasha") - # Or if you already have a session token - c = binding.Context(token="atg232342aa34324a") - # Or if you already have a valid cookie - c = binding.Context(cookie="splunkd_8089=...") - """ - - def __init__(self, handler=None, **kwargs): - self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), - cert_file=kwargs.get("cert_file"), context=kwargs.get("context"), - # Default to False for backward compat - retries=kwargs.get("retries", 0), retryDelay=kwargs.get("retryDelay", 10)) - self.token = kwargs.get("token", _NoAuthenticationToken) - if self.token is None: # In case someone explicitly passes token=None - self.token = _NoAuthenticationToken - self.scheme = kwargs.get("scheme", DEFAULT_SCHEME) - self.host = kwargs.get("host", DEFAULT_HOST) - self.port = int(kwargs.get("port", DEFAULT_PORT)) - self.authority = _authority(self.scheme, self.host, self.port) - self.namespace = namespace(**kwargs) - self.username = kwargs.get("username", "") - self.password = kwargs.get("password", "") - self.basic = kwargs.get("basic", False) - self.bearerToken = kwargs.get("splunkToken", "") - self.autologin = kwargs.get("autologin", False) - self.additional_headers = kwargs.get("headers", []) - - # Store any cookies in the self.http._cookies dict - if "cookie" in kwargs and kwargs['cookie'] not in [None, _NoAuthenticationToken]: - _parse_cookies(kwargs["cookie"], self.http._cookies) - - def get_cookies(self): - """Gets the dictionary of cookies from the ``HttpLib`` member of this instance. - - :return: Dictionary of cookies stored on the ``self.http``. - :rtype: ``dict`` - """ - return self.http._cookies - - def has_cookies(self): - """Returns true if the ``HttpLib`` member of this instance has auth token stored. - - :return: ``True`` if there is auth token present, else ``False`` - :rtype: ``bool`` - """ - auth_token_key = "splunkd_" - return any(auth_token_key in key for key in self.get_cookies().keys()) - - # Shared per-context request headers - @property - def _auth_headers(self): - """Headers required to authenticate a request. - - Assumes your ``Context`` already has a authentication token or - cookie, either provided explicitly or obtained by logging - into the Splunk instance. - - :returns: A list of 2-tuples containing key and value - """ - header = [] - if self.has_cookies(): - return [("Cookie", _make_cookie_header(list(self.get_cookies().items())))] - elif self.basic and (self.username and self.password): - token = f'Basic {b64encode(("%s:%s" % (self.username, self.password)).encode("utf-8")).decode("ascii")}' - elif self.bearerToken: - token = f'Bearer {self.bearerToken}' - elif self.token is _NoAuthenticationToken: - token = [] - else: - # Ensure the token is properly formatted - if self.token.startswith('Splunk '): - token = self.token - else: - token = f'Splunk {self.token}' - if token: - header.append(("Authorization", token)) - if self.get_cookies(): - header.append(("Cookie", _make_cookie_header(list(self.get_cookies().items())))) - - return header - - def connect(self): - """Returns an open connection (socket) to the Splunk instance. - - This method is used for writing bulk events to an index or similar tasks - where the overhead of opening a connection multiple times would be - prohibitive. - - :returns: A socket. - - **Example**:: - - import splunklib.binding as binding - c = binding.connect(...) - socket = c.connect() - socket.write("POST %s HTTP/1.1\\r\\n" % "some/path/to/post/to") - socket.write("Host: %s:%s\\r\\n" % (c.host, c.port)) - socket.write("Accept-Encoding: identity\\r\\n") - socket.write("Authorization: %s\\r\\n" % c.token) - socket.write("X-Splunk-Input-Mode: Streaming\\r\\n") - socket.write("\\r\\n") - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if self.scheme == "https": - sock = ssl.wrap_socket(sock) - sock.connect((socket.gethostbyname(self.host), self.port)) - return sock - - @_authentication - @_log_duration - def delete(self, path_segment, owner=None, app=None, sharing=None, **query): - """Performs a DELETE operation at the REST path segment with the given - namespace and query. - - This method is named to match the HTTP method. ``delete`` makes at least - one round trip to the server, one additional round trip for each 303 - status returned, and at most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - If *owner*, *app*, and *sharing* are omitted, this method uses the - default :class:`Context` namespace. All other keyword arguments are - included in the URL as query parameters. - - :raises AuthenticationError: Raised when the ``Context`` object is not - logged in. - :raises HTTPError: Raised when an error occurred in a GET operation from - *path_segment*. - :param path_segment: A REST path segment. - :type path_segment: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode of the namespace (optional). - :type sharing: ``string`` - :param query: All other keyword arguments, which are used as query - parameters. - :type query: ``string`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - c = binding.connect(...) - c.delete('saved/searches/boris') == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '1786'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 16:53:06 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'OK', - 'status': 200} - c.delete('nonexistant/path') # raises HTTPError - c.logout() - c.delete('apps/local') # raises AuthenticationError - """ - path = self.authority + self._abspath(path_segment, owner=owner, - app=app, sharing=sharing) - logger.debug("DELETE request to %s (body: %s)", path, mask_sensitive_data(query)) - response = self.http.delete(path, self._auth_headers, **query) - return response - - @_authentication - @_log_duration - def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, **query): - """Performs a GET operation from the REST path segment with the given - namespace and query. - - This method is named to match the HTTP method. ``get`` makes at least - one round trip to the server, one additional round trip for each 303 - status returned, and at most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - If *owner*, *app*, and *sharing* are omitted, this method uses the - default :class:`Context` namespace. All other keyword arguments are - included in the URL as query parameters. - - :raises AuthenticationError: Raised when the ``Context`` object is not - logged in. - :raises HTTPError: Raised when an error occurred in a GET operation from - *path_segment*. - :param path_segment: A REST path segment. - :type path_segment: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param headers: List of extra HTTP headers to send (optional). - :type headers: ``list`` of 2-tuples. - :param sharing: The sharing mode of the namespace (optional). - :type sharing: ``string`` - :param query: All other keyword arguments, which are used as query - parameters. - :type query: ``string`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - c = binding.connect(...) - c.get('apps/local') == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '26208'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 16:30:35 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'OK', - 'status': 200} - c.get('nonexistant/path') # raises HTTPError - c.logout() - c.get('apps/local') # raises AuthenticationError - """ - if headers is None: - headers = [] - - path = self.authority + self._abspath(path_segment, owner=owner, - app=app, sharing=sharing) - logger.debug("GET request to %s (body: %s)", path, mask_sensitive_data(query)) - all_headers = headers + self.additional_headers + self._auth_headers - response = self.http.get(path, all_headers, **query) - return response - - @_authentication - @_log_duration - def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, **query): - """Performs a POST operation from the REST path segment with the given - namespace and query. - - This method is named to match the HTTP method. ``post`` makes at least - one round trip to the server, one additional round trip for each 303 - status returned, and at most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - If *owner*, *app*, and *sharing* are omitted, this method uses the - default :class:`Context` namespace. All other keyword arguments are - included in the URL as query parameters. - - Some of Splunk's endpoints, such as ``receivers/simple`` and - ``receivers/stream``, require unstructured data in the POST body - and all metadata passed as GET-style arguments. If you provide - a ``body`` argument to ``post``, it will be used as the POST - body, and all other keyword arguments will be passed as - GET-style arguments in the URL. - - :raises AuthenticationError: Raised when the ``Context`` object is not - logged in. - :raises HTTPError: Raised when an error occurred in a GET operation from - *path_segment*. - :param path_segment: A REST path segment. - :type path_segment: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode of the namespace (optional). - :type sharing: ``string`` - :param headers: List of extra HTTP headers to send (optional). - :type headers: ``list`` of 2-tuples. - :param query: All other keyword arguments, which are used as query - parameters. - :param body: Parameters to be used in the post body. If specified, - any parameters in the query will be applied to the URL instead of - the body. If a dict is supplied, the key-value pairs will be form - encoded. If a string is supplied, the body will be passed through - in the request unchanged. - :type body: ``dict`` or ``str`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - c = binding.connect(...) - c.post('saved/searches', name='boris', - search='search * earliest=-1m | head 1') == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '10455'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 16:46:06 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'Created', - 'status': 201} - c.post('nonexistant/path') # raises HTTPError - c.logout() - # raises AuthenticationError: - c.post('saved/searches', name='boris', - search='search * earliest=-1m | head 1') - """ - if headers is None: - headers = [] - - path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - - logger.debug("POST request to %s (body: %s)", path, mask_sensitive_data(query)) - all_headers = headers + self.additional_headers + self._auth_headers - response = self.http.post(path, all_headers, **query) - return response - - @_authentication - @_log_duration - def request(self, path_segment, method="GET", headers=None, body={}, - owner=None, app=None, sharing=None): - """Issues an arbitrary HTTP request to the REST path segment. - - This method is named to match ``httplib.request``. This function - makes a single round trip to the server. - - If *owner*, *app*, and *sharing* are omitted, this method uses the - default :class:`Context` namespace. All other keyword arguments are - included in the URL as query parameters. - - :raises AuthenticationError: Raised when the ``Context`` object is not - logged in. - :raises HTTPError: Raised when an error occurred in a GET operation from - *path_segment*. - :param path_segment: A REST path segment. - :type path_segment: ``string`` - :param method: The HTTP method to use (optional). - :type method: ``string`` - :param headers: List of extra HTTP headers to send (optional). - :type headers: ``list`` of 2-tuples. - :param body: Content of the HTTP request (optional). - :type body: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode of the namespace (optional). - :type sharing: ``string`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - c = binding.connect(...) - c.request('saved/searches', method='GET') == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '46722'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 17:24:19 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'OK', - 'status': 200} - c.request('nonexistant/path', method='GET') # raises HTTPError - c.logout() - c.get('apps/local') # raises AuthenticationError - """ - if headers is None: - headers = [] - - path = self.authority \ - + self._abspath(path_segment, owner=owner, - app=app, sharing=sharing) - - all_headers = headers + self.additional_headers + self._auth_headers - logger.debug("%s request to %s (headers: %s, body: %s)", - method, path, str(mask_sensitive_data(dict(all_headers))), mask_sensitive_data(body)) - if body: - body = _encode(**body) - - if method == "GET": - path = path + UrlEncoded('?' + body, skip_encode=True) - message = {'method': method, - 'headers': all_headers} - else: - message = {'method': method, - 'headers': all_headers, - 'body': body} - else: - message = {'method': method, - 'headers': all_headers} - - response = self.http.request(path, message) - - return response - - def login(self): - """Logs into the Splunk instance referred to by the :class:`Context` - object. - - Unless a ``Context`` is created with an explicit authentication token - (probably obtained by logging in from a different ``Context`` object) - you must call :meth:`login` before you can issue requests. - The authentication token obtained from the server is stored in the - ``token`` field of the ``Context`` object. - - :raises AuthenticationError: Raised when login fails. - :returns: The ``Context`` object, so you can chain calls. - - **Example**:: - - import splunklib.binding as binding - c = binding.Context(...).login() - # Then issue requests... - """ - - if self.has_cookies() and \ - (not self.username and not self.password): - # If we were passed session cookie(s), but no username or - # password, then login is a nop, since we're automatically - # logged in. - return - - if self.token is not _NoAuthenticationToken and \ - (not self.username and not self.password): - # If we were passed a session token, but no username or - # password, then login is a nop, since we're automatically - # logged in. - return - - if self.basic and (self.username and self.password): - # Basic auth mode requested, so this method is a nop as long - # as credentials were passed in. - return - - if self.bearerToken: - # Bearer auth mode requested, so this method is a nop as long - # as authentication token was passed in. - return - # Only try to get a token and updated cookie if username & password are specified - try: - response = self.http.post( - self.authority + self._abspath("/services/auth/login"), - username=self.username, - password=self.password, - headers=self.additional_headers, - cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header - - body = response.body.read() - session = XML(body).findtext("./sessionKey") - self.token = f"Splunk {session}" - return self - except HTTPError as he: - if he.status == 401: - raise AuthenticationError("Login failed.", he) - else: - raise - - def logout(self): - """Forgets the current session token, and cookies.""" - self.token = _NoAuthenticationToken - self.http._cookies = {} - return self - - def _abspath(self, path_segment, - owner=None, app=None, sharing=None): - """Qualifies *path_segment* into an absolute path for a URL. - - If *path_segment* is already absolute, returns it unchanged. - If *path_segment* is relative, then qualifies it with either - the provided namespace arguments or the ``Context``'s default - namespace. Any forbidden characters in *path_segment* are URL - encoded. This function has no network activity. - - Named to be consistent with RFC2396_. - - .. _RFC2396: http://www.ietf.org/rfc/rfc2396.txt - - :param path_segment: A relative or absolute URL path segment. - :type path_segment: ``string`` - :param owner, app, sharing: Components of a namespace (defaults - to the ``Context``'s namespace if all - three are omitted) - :type owner, app, sharing: ``string`` - :return: A ``UrlEncoded`` (a subclass of ``str``). - :rtype: ``string`` - - **Example**:: - - import splunklib.binding as binding - c = binding.connect(owner='boris', app='search', sharing='user') - c._abspath('/a/b/c') == '/a/b/c' - c._abspath('/a/b c/d') == '/a/b%20c/d' - c._abspath('apps/local/search') == \ - '/servicesNS/boris/search/apps/local/search' - c._abspath('apps/local/search', sharing='system') == \ - '/servicesNS/nobody/system/apps/local/search' - url = c.authority + c._abspath('apps/local/sharing') - """ - skip_encode = isinstance(path_segment, UrlEncoded) - # If path_segment is absolute, escape all forbidden characters - # in it and return it. - if path_segment.startswith('/'): - return UrlEncoded(path_segment, skip_encode=skip_encode) - - # path_segment is relative, so we need a namespace to build an - # absolute path. - if owner or app or sharing: - ns = namespace(owner=owner, app=app, sharing=sharing) - else: - ns = self.namespace - - # If no app or owner are specified, then use the /services - # endpoint. Otherwise, use /servicesNS with the specified - # namespace. If only one of app and owner is specified, use - # '-' for the other. - if ns.app is None and ns.owner is None: - return UrlEncoded(f"/services/{path_segment}", skip_encode=skip_encode) - - oname = "nobody" if ns.owner is None else ns.owner - aname = "system" if ns.app is None else ns.app - path = UrlEncoded(f"/servicesNS/{oname}/{aname}/{path_segment}", skip_encode=skip_encode) - return path - - -def connect(**kwargs): - """This function returns an authenticated :class:`Context` object. - - This function is a shorthand for calling :meth:`Context.login`. - - This function makes one round trip to the server. - - :param host: The host name (the default is "localhost"). - :type host: ``string`` - :param port: The port number (the default is 8089). - :type port: ``integer`` - :param scheme: The scheme for accessing the service (the default is "https"). - :type scheme: "https" or "http" - :param owner: The owner context of the namespace (the default is "None"). - :type owner: ``string`` - :param app: The app context of the namespace (the default is "None"). - :type app: ``string`` - :param sharing: The sharing mode for the namespace (the default is "user"). - :type sharing: "global", "system", "app", or "user" - :param token: The current session token (optional). Session tokens can be - shared across multiple service instances. - :type token: ``string`` - :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. - This parameter is only supported for Splunk 6.2+. - :type cookie: ``string`` - :param username: The Splunk account username, which is used to - authenticate the Splunk instance. - :type username: ``string`` - :param password: The password for the Splunk account. - :type password: ``string`` - :param headers: List of extra HTTP headers to send (optional). - :type headers: ``list`` of 2-tuples. - :param autologin: When ``True``, automatically tries to log in again if the - session terminates. - :type autologin: ``Boolean`` - :return: An initialized :class:`Context` instance. - - **Example**:: - - import splunklib.binding as binding - c = binding.connect(...) - response = c.get("apps/local") - """ - c = Context(**kwargs) - c.login() - return c - - -# Note: the error response schema supports multiple messages but we only -# return the first, although we do return the body so that an exception -# handler that wants to read multiple messages can do so. -class HTTPError(Exception): - """This exception is raised for HTTP responses that return an error.""" - - def __init__(self, response, _message=None): - status = response.status - reason = response.reason - body = response.body.read() - try: - detail = XML(body).findtext("./messages/msg") - except ParseError: - detail = body - detail_formatted = "" if detail is None else f" -- {detail}" - message = f"HTTP {status} {reason}{detail_formatted}" - Exception.__init__(self, _message or message) - self.status = status - self.reason = reason - self.headers = response.headers - self.body = body - self._response = response - - -class AuthenticationError(HTTPError): - """Raised when a login request to Splunk fails. - - If your username was unknown or you provided an incorrect password - in a call to :meth:`Context.login` or :meth:`splunklib.client.Service.login`, - this exception is raised. - """ - - def __init__(self, message, cause): - # Put the body back in the response so that HTTPError's constructor can - # read it again. - cause._response.body = BytesIO(cause.body) - - HTTPError.__init__(self, cause._response, message) - - -# -# The HTTP interface used by the Splunk binding layer abstracts the underlying -# HTTP library using request & response 'messages' which are implemented as -# dictionaries with the following structure: -# -# # HTTP request message (only method required) -# request { -# method : str, -# headers? : [(str, str)*], -# body? : str, -# } -# -# # HTTP response message (all keys present) -# response { -# status : int, -# reason : str, -# headers : [(str, str)*], -# body : file, -# } -# - -# Encode the given kwargs as a query string. This wrapper will also _encode -# a list value as a sequence of assignments to the corresponding arg name, -# for example an argument such as 'foo=[1,2,3]' will be encoded as -# 'foo=1&foo=2&foo=3'. -def _encode(**kwargs): - items = [] - for key, value in kwargs.items(): - if isinstance(value, list): - items.extend([(key, item) for item in value]) - else: - items.append((key, value)) - return parse.urlencode(items) - - -# Crack the given url into (scheme, host, port, path) -def _spliturl(url): - parsed_url = parse.urlparse(url) - host = parsed_url.hostname - port = parsed_url.port - path = '?'.join((parsed_url.path, parsed_url.query)) if parsed_url.query else parsed_url.path - # Strip brackets if its an IPv6 address - if host.startswith('[') and host.endswith(']'): host = host[1:-1] - if port is None: port = DEFAULT_PORT - return parsed_url.scheme, host, port, path - - -# Given an HTTP request handler, this wrapper objects provides a related -# family of convenience methods built using that handler. -class HttpLib: - """A set of convenient methods for making HTTP calls. - - ``HttpLib`` provides a general :meth:`request` method, and :meth:`delete`, - :meth:`post`, and :meth:`get` methods for the three HTTP methods that Splunk - uses. - - By default, ``HttpLib`` uses Python's built-in ``httplib`` library, - but you can replace it by passing your own handling function to the - constructor for ``HttpLib``. - - The handling function should have the type: - - ``handler(`url`, `request_dict`) -> response_dict`` - - where `url` is the URL to make the request to (including any query and - fragment sections) as a dictionary with the following keys: - - - method: The method for the request, typically ``GET``, ``POST``, or ``DELETE``. - - - headers: A list of pairs specifying the HTTP headers (for example: ``[('key': value), ...]``). - - - body: A string containing the body to send with the request (this string - should default to ''). - - and ``response_dict`` is a dictionary with the following keys: - - - status: An integer containing the HTTP status code (such as 200 or 404). - - - reason: The reason phrase, if any, returned by the server. - - - headers: A list of pairs containing the response headers (for example, ``[('key': value), ...]``). - - - body: A stream-like object supporting ``read(size=None)`` and ``close()`` - methods to get the body of the response. - - The response dictionary is returned directly by ``HttpLib``'s methods with - no further processing. By default, ``HttpLib`` calls the :func:`handler` function - to get a handler function. - - If using the default handler, SSL verification can be disabled by passing verify=False. - """ - - def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None, retries=0, - retryDelay=10): - if custom_handler is None: - self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file, context=context) - else: - self.handler = custom_handler - self._cookies = {} - self.retries = retries - self.retryDelay = retryDelay - - def delete(self, url, headers=None, **kwargs): - """Sends a DELETE request to a URL. - - :param url: The URL. - :type url: ``string`` - :param headers: A list of pairs specifying the headers for the HTTP - response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). - :type headers: ``list`` - :param kwargs: Additional keyword arguments (optional). These arguments - are interpreted as the query part of the URL. The order of keyword - arguments is not preserved in the request, but the keywords and - their arguments will be URL encoded. - :type kwargs: ``dict`` - :returns: A dictionary describing the response (see :class:`HttpLib` for - its structure). - :rtype: ``dict`` - """ - if headers is None: headers = [] - if kwargs: - # url is already a UrlEncoded. We have to manually declare - # the query to be encoded or it will get automatically URL - # encoded by being appended to url. - url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) - message = { - 'method': "DELETE", - 'headers': headers, - } - return self.request(url, message) - - def get(self, url, headers=None, **kwargs): - """Sends a GET request to a URL. - - :param url: The URL. - :type url: ``string`` - :param headers: A list of pairs specifying the headers for the HTTP - response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). - :type headers: ``list`` - :param kwargs: Additional keyword arguments (optional). These arguments - are interpreted as the query part of the URL. The order of keyword - arguments is not preserved in the request, but the keywords and - their arguments will be URL encoded. - :type kwargs: ``dict`` - :returns: A dictionary describing the response (see :class:`HttpLib` for - its structure). - :rtype: ``dict`` - """ - if headers is None: headers = [] - if kwargs: - # url is already a UrlEncoded. We have to manually declare - # the query to be encoded or it will get automatically URL - # encoded by being appended to url. - url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) - return self.request(url, {'method': "GET", 'headers': headers}) - - def post(self, url, headers=None, **kwargs): - """Sends a POST request to a URL. - - :param url: The URL. - :type url: ``string`` - :param headers: A list of pairs specifying the headers for the HTTP - response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). - :type headers: ``list`` - :param kwargs: Additional keyword arguments (optional). If the argument - is ``body``, the value is used as the body for the request, and the - keywords and their arguments will be URL encoded. If there is no - ``body`` keyword argument, all the keyword arguments are encoded - into the body of the request in the format ``x-www-form-urlencoded``. - :type kwargs: ``dict`` - :returns: A dictionary describing the response (see :class:`HttpLib` for - its structure). - :rtype: ``dict`` - """ - if headers is None: headers = [] - - # We handle GET-style arguments and an unstructured body. This is here - # to support the receivers/stream endpoint. - if 'body' in kwargs: - # We only use application/x-www-form-urlencoded if there is no other - # Content-Type header present. This can happen in cases where we - # send requests as application/json, e.g. for KV Store. - if len([x for x in headers if x[0].lower() == "content-type"]) == 0: - headers.append(("Content-Type", "application/x-www-form-urlencoded")) - - body = kwargs.pop('body') - if isinstance(body, dict): - body = _encode(**body).encode('utf-8') - if len(kwargs) > 0: - url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) - else: - body = _encode(**kwargs).encode('utf-8') - message = { - 'method': "POST", - 'headers': headers, - 'body': body - } - return self.request(url, message) - - def request(self, url, message, **kwargs): - """Issues an HTTP request to a URL. - - :param url: The URL. - :type url: ``string`` - :param message: A dictionary with the format as described in - :class:`HttpLib`. - :type message: ``dict`` - :param kwargs: Additional keyword arguments (optional). These arguments - are passed unchanged to the handler. - :type kwargs: ``dict`` - :returns: A dictionary describing the response (see :class:`HttpLib` for - its structure). - :rtype: ``dict`` - """ - while True: - try: - response = self.handler(url, message, **kwargs) - break - except Exception: - if self.retries <= 0: - raise - else: - time.sleep(self.retryDelay) - self.retries -= 1 - response = record(response) - if 400 <= response.status: - raise HTTPError(response) - - # Update the cookie with any HTTP request - # Initially, assume list of 2-tuples - key_value_tuples = response.headers - # If response.headers is a dict, get the key-value pairs as 2-tuples - # this is the case when using urllib2 - if isinstance(response.headers, dict): - key_value_tuples = list(response.headers.items()) - for key, value in key_value_tuples: - if key.lower() == "set-cookie": - _parse_cookies(value, self._cookies) - - return response - - -# Converts an httplib response into a file-like object. -class ResponseReader(io.RawIOBase): - """This class provides a file-like interface for :class:`httplib` responses. - - The ``ResponseReader`` class is intended to be a layer to unify the different - types of HTTP libraries used with this SDK. This class also provides a - preview of the stream and a few useful predicates. - """ - - # For testing, you can use a StringIO as the argument to - # ``ResponseReader`` instead of an ``httplib.HTTPResponse``. It - # will work equally well. - def __init__(self, response, connection=None): - self._response = response - self._connection = connection - self._buffer = b'' - - def __str__(self): - return str(self.read(), 'UTF-8') - - @property - def empty(self): - """Indicates whether there is any more data in the response.""" - return self.peek(1) == b"" - - def peek(self, size): - """Nondestructively retrieves a given number of characters. - - The next :meth:`read` operation behaves as though this method was never - called. - - :param size: The number of characters to retrieve. - :type size: ``integer`` - """ - c = self.read(size) - self._buffer = self._buffer + c - return c - - def close(self): - """Closes this response.""" - if self._connection: - self._connection.close() - self._response.close() - - def read(self, size=None): - """Reads a given number of characters from the response. - - :param size: The number of characters to read, or "None" to read the - entire response. - :type size: ``integer`` or "None" - - """ - r = self._buffer - self._buffer = b'' - if size is not None: - size -= len(r) - r = r + self._response.read(size) - return r - - def readable(self): - """ Indicates that the response reader is readable.""" - return True - - def readinto(self, byte_array): - """ Read data into a byte array, upto the size of the byte array. - - :param byte_array: A byte array/memory view to pour bytes into. - :type byte_array: ``bytearray`` or ``memoryview`` - - """ - max_size = len(byte_array) - data = self.read(max_size) - bytes_read = len(data) - byte_array[:bytes_read] = data - return bytes_read - - -def handler(key_file=None, cert_file=None, timeout=None, verify=False, context=None): - """This class returns an instance of the default HTTP request handler using - the values you provide. - - :param `key_file`: A path to a PEM (Privacy Enhanced Mail) formatted file containing your private key (optional). - :type key_file: ``string`` - :param `cert_file`: A path to a PEM (Privacy Enhanced Mail) formatted file containing a certificate chain file (optional). - :type cert_file: ``string`` - :param `timeout`: The request time-out period, in seconds (optional). - :type timeout: ``integer`` or "None" - :param `verify`: Set to False to disable SSL verification on https connections. - :type verify: ``Boolean`` - :param `context`: The SSLContext that can is used with the HTTPSConnection when verify=True is enabled and context is specified - :type context: ``SSLContext` - """ - - def connect(scheme, host, port): - kwargs = {} - if timeout is not None: kwargs['timeout'] = timeout - if scheme == "http": - return client.HTTPConnection(host, port, **kwargs) - if scheme == "https": - if key_file is not None: kwargs['key_file'] = key_file - if cert_file is not None: kwargs['cert_file'] = cert_file - - if not verify: - kwargs['context'] = ssl._create_unverified_context() - elif context: - # verify is True in elif branch and context is not None - kwargs['context'] = context - - return client.HTTPSConnection(host, port, **kwargs) - raise ValueError(f"unsupported scheme: {scheme}") - - def request(url, message, **kwargs): - scheme, host, port, path = _spliturl(url) - body = message.get("body", "") - head = { - "Content-Length": str(len(body)), - "Host": host, - "User-Agent": "splunk-sdk-python/%s" % __version__, - "Accept": "*/*", - "Connection": "Close", - } # defaults - for key, value in message["headers"]: - head[key] = value - method = message.get("method", "GET") - - connection = connect(scheme, host, port) - is_keepalive = False - try: - connection.request(method, path, body, head) - if timeout is not None: - connection.sock.settimeout(timeout) - response = connection.getresponse() - is_keepalive = "keep-alive" in response.getheader("connection", default="close").lower() - finally: - if not is_keepalive: - connection.close() - - return { - "status": response.status, - "reason": response.reason, - "headers": response.getheaders(), - "body": ResponseReader(response, connection if is_keepalive else None), - } - - return request +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The **splunklib.binding** module provides a low-level binding interface to the +`Splunk REST API `_. + +This module handles the wire details of calling the REST API, such as +authentication tokens, prefix paths, URL encoding, and so on. Actual path +segments, ``GET`` and ``POST`` arguments, and the parsing of responses is left +to the user. + +If you want a friendlier interface to the Splunk REST API, use the +:mod:`splunklib.client` module. +""" + +import io +import json +import logging +import socket +import ssl +import time +from base64 import b64encode +from contextlib import contextmanager +from datetime import datetime +from functools import wraps +from io import BytesIO +from urllib import parse +from http import client +from http.cookies import SimpleCookie +from xml.etree.ElementTree import XML, ParseError +from splunklib.data import record +from splunklib import __version__ + + +logger = logging.getLogger(__name__) + +__all__ = [ + "AuthenticationError", + "connect", + "Context", + "handler", + "HTTPError", + "UrlEncoded", + "_encode", + "_make_cookie_header", + "_NoAuthenticationToken", + "namespace" +] + +SENSITIVE_KEYS = ['Authorization', 'Cookie', 'action.email.auth_password', 'auth', 'auth_password', 'clear_password', 'clientId', + 'crc-salt', 'encr_password', 'oldpassword', 'passAuth', 'password', 'session', 'suppressionKey', + 'token'] + +# If you change these, update the docstring +# on _authority as well. +DEFAULT_HOST = "localhost" +DEFAULT_PORT = "8089" +DEFAULT_SCHEME = "https" + + +def _log_duration(f): + @wraps(f) + def new_f(*args, **kwargs): + start_time = datetime.now() + val = f(*args, **kwargs) + end_time = datetime.now() + logger.debug("Operation took %s", end_time - start_time) + return val + + return new_f + + +def mask_sensitive_data(data): + ''' + Masked sensitive fields data for logging purpose + ''' + if not isinstance(data, dict): + try: + data = json.loads(data) + except Exception as ex: + return data + + # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type + if not isinstance(data, dict): + return data + mdata = {} + for k, v in data.items(): + if k in SENSITIVE_KEYS: + mdata[k] = "******" + else: + mdata[k] = mask_sensitive_data(v) + return mdata + + +def _parse_cookies(cookie_str, dictionary): + """Tries to parse any key-value pairs of cookies in a string, + then updates the the dictionary with any key-value pairs found. + + **Example**:: + + dictionary = {} + _parse_cookies('my=value', dictionary) + # Now the following is True + dictionary['my'] == 'value' + + :param cookie_str: A string containing "key=value" pairs from an HTTP "Set-Cookie" header. + :type cookie_str: ``str`` + :param dictionary: A dictionary to update with any found key-value pairs. + :type dictionary: ``dict`` + """ + parsed_cookie = SimpleCookie(cookie_str) + for cookie in parsed_cookie.values(): + dictionary[cookie.key] = cookie.coded_value + + +def _make_cookie_header(cookies): + """ + Takes a list of 2-tuples of key-value pairs of + cookies, and returns a valid HTTP ``Cookie`` + header. + + **Example**:: + + header = _make_cookie_header([("key", "value"), ("key_2", "value_2")]) + # Now the following is True + header == "key=value; key_2=value_2" + + :param cookies: A list of 2-tuples of cookie key-value pairs. + :type cookies: ``list`` of 2-tuples + :return: ``str` An HTTP header cookie string. + :rtype: ``str`` + """ + return "; ".join(f"{key}={value}" for key, value in cookies) + + +# Singleton values to eschew None +class _NoAuthenticationToken: + """The value stored in a :class:`Context` or :class:`splunklib.client.Service` + class that is not logged in. + + If a ``Context`` or ``Service`` object is created without an authentication + token, and there has not yet been a call to the ``login`` method, the token + field of the ``Context`` or ``Service`` object is set to + ``_NoAuthenticationToken``. + + Likewise, after a ``Context`` or ``Service`` object has been logged out, the + token is set to this value again. + """ + + +class UrlEncoded(str): + """This class marks URL-encoded strings. + It should be considered an SDK-private implementation detail. + + Manually tracking whether strings are URL encoded can be difficult. Avoid + calling ``urllib.quote`` to replace special characters with escapes. When + you receive a URL-encoded string, *do* use ``urllib.unquote`` to replace + escapes with single characters. Then, wrap any string you want to use as a + URL in ``UrlEncoded``. Note that because the ``UrlEncoded`` class is + idempotent, making multiple calls to it is OK. + + ``UrlEncoded`` objects are identical to ``str`` objects (including being + equal if their contents are equal) except when passed to ``UrlEncoded`` + again. + + ``UrlEncoded`` removes the ``str`` type support for interpolating values + with ``%`` (doing that raises a ``TypeError``). There is no reliable way to + encode values this way, so instead, interpolate into a string, quoting by + hand, and call ``UrlEncode`` with ``skip_encode=True``. + + **Example**:: + + import urllib + UrlEncoded(f'{scheme}://{urllib.quote(host)}', skip_encode=True) + + If you append ``str`` strings and ``UrlEncoded`` strings, the result is also + URL encoded. + + **Example**:: + + UrlEncoded('ab c') + 'de f' == UrlEncoded('ab cde f') + 'ab c' + UrlEncoded('de f') == UrlEncoded('ab cde f') + """ + + def __new__(self, val='', skip_encode=False, encode_slash=False): + if isinstance(val, UrlEncoded): + # Don't urllib.quote something already URL encoded. + return val + if skip_encode: + return str.__new__(self, val) + if encode_slash: + return str.__new__(self, parse.quote_plus(val)) + # When subclassing str, just call str.__new__ method + # with your class and the value you want to have in the + # new string. + return str.__new__(self, parse.quote(val)) + + def __add__(self, other): + """self + other + + If *other* is not a ``UrlEncoded``, URL encode it before + adding it. + """ + if isinstance(other, UrlEncoded): + return UrlEncoded(str.__add__(self, other), skip_encode=True) + + return UrlEncoded(str.__add__(self, parse.quote(other)), skip_encode=True) + + def __radd__(self, other): + """other + self + + If *other* is not a ``UrlEncoded``, URL _encode it before + adding it. + """ + if isinstance(other, UrlEncoded): + return UrlEncoded(str.__radd__(self, other), skip_encode=True) + + return UrlEncoded(str.__add__(parse.quote(other), self), skip_encode=True) + + def __mod__(self, fields): + """Interpolation into ``UrlEncoded``s is disabled. + + If you try to write ``UrlEncoded("%s") % "abc", will get a + ``TypeError``. + """ + raise TypeError("Cannot interpolate into a UrlEncoded object.") + + def __repr__(self): + return f"UrlEncoded({repr(parse.unquote(str(self)))})" + + +@contextmanager +def _handle_auth_error(msg): + """Handle re-raising HTTP authentication errors as something clearer. + + If an ``HTTPError`` is raised with status 401 (access denied) in + the body of this context manager, re-raise it as an + ``AuthenticationError`` instead, with *msg* as its message. + + This function adds no round trips to the server. + + :param msg: The message to be raised in ``AuthenticationError``. + :type msg: ``str`` + + **Example**:: + + with _handle_auth_error("Your login failed."): + ... # make an HTTP request + """ + try: + yield + except HTTPError as he: + if he.status == 401: + raise AuthenticationError(msg, he) + else: + raise + + +def _authentication(request_fun): + """Decorator to handle autologin and authentication errors. + + *request_fun* is a function taking no arguments that needs to + be run with this ``Context`` logged into Splunk. + + ``_authentication``'s behavior depends on whether the + ``autologin`` field of ``Context`` is set to ``True`` or + ``False``. If it's ``False``, then ``_authentication`` + aborts if the ``Context`` is not logged in, and raises an + ``AuthenticationError`` if an ``HTTPError`` of status 401 is + raised in *request_fun*. If it's ``True``, then + ``_authentication`` will try at all sensible places to + log in before issuing the request. + + If ``autologin`` is ``False``, ``_authentication`` makes + one roundtrip to the server if the ``Context`` is logged in, + or zero if it is not. If ``autologin`` is ``True``, it's less + deterministic, and may make at most three roundtrips (though + that would be a truly pathological case). + + :param request_fun: A function of no arguments encapsulating + the request to make to the server. + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(..., autologin=True) + c.logout() + def f(): + c.get("/services") + return 42 + print(_authentication(f)) + """ + + @wraps(request_fun) + def wrapper(self, *args, **kwargs): + if self.token is _NoAuthenticationToken and not self.has_cookies(): + # Not yet logged in. + if self.autologin and self.username and self.password: + # This will throw an uncaught + # AuthenticationError if it fails. + self.login() + else: + # Try the request anyway without authentication. + # Most requests will fail. Some will succeed, such as + # 'GET server/info'. + with _handle_auth_error("Request aborted: not logged in."): + return request_fun(self, *args, **kwargs) + try: + # Issue the request + return request_fun(self, *args, **kwargs) + except HTTPError as he: + if he.status == 401 and self.autologin: + # Authentication failed. Try logging in, and then + # rerunning the request. If either step fails, throw + # an AuthenticationError and give up. + with _handle_auth_error("Autologin failed."): + self.login() + with _handle_auth_error("Authentication Failed! If session token is used, it seems to have been expired."): + return request_fun(self, *args, **kwargs) + elif he.status == 401 and not self.autologin: + raise AuthenticationError( + "Request failed: Session is not logged in.", he) + else: + raise + + return wrapper + + +def _authority(scheme=DEFAULT_SCHEME, host=DEFAULT_HOST, port=DEFAULT_PORT): + """Construct a URL authority from the given *scheme*, *host*, and *port*. + + Named in accordance with RFC2396_, which defines URLs as:: + + ://? + + .. _RFC2396: http://www.ietf.org/rfc/rfc2396.txt + + So ``https://localhost:8000/a/b/b?boris=hilda`` would be parsed as:: + + scheme := https + authority := localhost:8000 + path := /a/b/c + query := boris=hilda + + :param scheme: URL scheme (the default is "https") + :type scheme: "http" or "https" + :param host: The host name (the default is "localhost") + :type host: string + :param port: The port number (the default is 8089) + :type port: integer + :return: The URL authority. + :rtype: UrlEncoded (subclass of ``str``) + + **Example**:: + + _authority() == "https://localhost:8089" + + _authority(host="splunk.utopia.net") == "https://splunk.utopia.net:8089" + + _authority(host="2001:0db8:85a3:0000:0000:8a2e:0370:7334") == \ + "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089" + + _authority(scheme="http", host="splunk.utopia.net", port="471") == \ + "http://splunk.utopia.net:471" + + """ + # check if host is an IPv6 address and not enclosed in [ ] + if ':' in host and not (host.startswith('[') and host.endswith(']')): + # IPv6 addresses must be enclosed in [ ] in order to be well + # formed. + host = '[' + host + ']' + return UrlEncoded(f"{scheme}://{host}:{port}", skip_encode=True) + + +# kwargs: sharing, owner, app +def namespace(sharing=None, owner=None, app=None, **kwargs): + """This function constructs a Splunk namespace. + + Every Splunk resource belongs to a namespace. The namespace is specified by + the pair of values ``owner`` and ``app`` and is governed by a ``sharing`` mode. + The possible values for ``sharing`` are: "user", "app", "global" and "system", + which map to the following combinations of ``owner`` and ``app`` values: + + "user" => {owner}, {app} + + "app" => nobody, {app} + + "global" => nobody, {app} + + "system" => nobody, system + + "nobody" is a special user name that basically means no user, and "system" + is the name reserved for system resources. + + "-" is a wildcard that can be used for both ``owner`` and ``app`` values and + refers to all users and all apps, respectively. + + In general, when you specify a namespace you can specify any combination of + these three values and the library will reconcile the triple, overriding the + provided values as appropriate. + + Finally, if no namespacing is specified the library will make use of the + ``/services`` branch of the REST API, which provides a namespaced view of + Splunk resources equivelent to using ``owner={currentUser}`` and + ``app={defaultApp}``. + + The ``namespace`` function returns a representation of the namespace from + reconciling the values you provide. It ignores any keyword arguments other + than ``owner``, ``app``, and ``sharing``, so you can provide ``dicts`` of + configuration information without first having to extract individual keys. + + :param sharing: The sharing mode (the default is "user"). + :type sharing: "system", "global", "app", or "user" + :param owner: The owner context (the default is "None"). + :type owner: ``string`` + :param app: The app context (the default is "None"). + :type app: ``string`` + :returns: A :class:`splunklib.data.Record` containing the reconciled + namespace. + + **Example**:: + + import splunklib.binding as binding + n = binding.namespace(sharing="user", owner="boris", app="search") + n = binding.namespace(sharing="global", app="search") + """ + if sharing in ["system"]: + return record({'sharing': sharing, 'owner': "nobody", 'app': "system"}) + if sharing in ["global", "app"]: + return record({'sharing': sharing, 'owner': "nobody", 'app': app}) + if sharing in ["user", None]: + return record({'sharing': sharing, 'owner': owner, 'app': app}) + raise ValueError("Invalid value for argument: 'sharing'") + + +class Context: + """This class represents a context that encapsulates a splunkd connection. + + The ``Context`` class encapsulates the details of HTTP requests, + authentication, a default namespace, and URL prefixes to simplify access to + the REST API. + + After creating a ``Context`` object, you must call its :meth:`login` + method before you can issue requests to splunkd. Or, use the :func:`connect` + function to create an already-authenticated ``Context`` object. You can + provide a session token explicitly (the same token can be shared by multiple + ``Context`` objects) to provide authentication. + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param verify: Enable (True) or disable (False) SSL verification for https connections. + :type verify: ``Boolean`` + :param sharing: The sharing mode for the namespace (the default is "user"). + :type sharing: "global", "system", "app", or "user" + :param owner: The owner context of the namespace (optional, the default is "None"). + :type owner: ``string`` + :param app: The app context of the namespace (optional, the default is "None"). + :type app: ``string`` + :param token: A session token. When provided, you don't need to call :meth:`login`. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param username: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param password: The password for the Splunk account. + :type password: ``string`` + :param splunkToken: Splunk authentication token + :type splunkToken: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param retires: Number of retries for each HTTP connection (optional, the default is 0). + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER AND BLOCK THE + CURRENT THREAD WHILE RETRYING. + :type retries: ``int`` + :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryDelay: ``int`` (in seconds) + :param handler: The HTTP request handler (optional). + :returns: A ``Context`` instance. + + **Example**:: + + import splunklib.binding as binding + c = binding.Context(username="boris", password="natasha", ...) + c.login() + # Or equivalently + c = binding.connect(username="boris", password="natasha") + # Or if you already have a session token + c = binding.Context(token="atg232342aa34324a") + # Or if you already have a valid cookie + c = binding.Context(cookie="splunkd_8089=...") + """ + + def __init__(self, handler=None, **kwargs): + self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), + cert_file=kwargs.get("cert_file"), context=kwargs.get("context"), + # Default to False for backward compat + retries=kwargs.get("retries", 0), retryDelay=kwargs.get("retryDelay", 10)) + self.token = kwargs.get("token", _NoAuthenticationToken) + if self.token is None: # In case someone explicitly passes token=None + self.token = _NoAuthenticationToken + self.scheme = kwargs.get("scheme", DEFAULT_SCHEME) + self.host = kwargs.get("host", DEFAULT_HOST) + self.port = int(kwargs.get("port", DEFAULT_PORT)) + self.authority = _authority(self.scheme, self.host, self.port) + self.namespace = namespace(**kwargs) + self.username = kwargs.get("username", "") + self.password = kwargs.get("password", "") + self.basic = kwargs.get("basic", False) + self.bearerToken = kwargs.get("splunkToken", "") + self.autologin = kwargs.get("autologin", False) + self.additional_headers = kwargs.get("headers", []) + + # Store any cookies in the self.http._cookies dict + if "cookie" in kwargs and kwargs['cookie'] not in [None, _NoAuthenticationToken]: + _parse_cookies(kwargs["cookie"], self.http._cookies) + + def get_cookies(self): + """Gets the dictionary of cookies from the ``HttpLib`` member of this instance. + + :return: Dictionary of cookies stored on the ``self.http``. + :rtype: ``dict`` + """ + return self.http._cookies + + def has_cookies(self): + """Returns true if the ``HttpLib`` member of this instance has auth token stored. + + :return: ``True`` if there is auth token present, else ``False`` + :rtype: ``bool`` + """ + auth_token_key = "splunkd_" + return any(auth_token_key in key for key in self.get_cookies().keys()) + + # Shared per-context request headers + @property + def _auth_headers(self): + """Headers required to authenticate a request. + + Assumes your ``Context`` already has a authentication token or + cookie, either provided explicitly or obtained by logging + into the Splunk instance. + + :returns: A list of 2-tuples containing key and value + """ + header = [] + if self.has_cookies(): + return [("Cookie", _make_cookie_header(list(self.get_cookies().items())))] + elif self.basic and (self.username and self.password): + token = f'Basic {b64encode(("%s:%s" % (self.username, self.password)).encode("utf-8")).decode("ascii")}' + elif self.bearerToken: + token = f'Bearer {self.bearerToken}' + elif self.token is _NoAuthenticationToken: + token = [] + else: + # Ensure the token is properly formatted + if self.token.startswith('Splunk '): + token = self.token + else: + token = f'Splunk {self.token}' + if token: + header.append(("Authorization", token)) + if self.get_cookies(): + header.append(("Cookie", _make_cookie_header(list(self.get_cookies().items())))) + + return header + + def connect(self): + """Returns an open connection (socket) to the Splunk instance. + + This method is used for writing bulk events to an index or similar tasks + where the overhead of opening a connection multiple times would be + prohibitive. + + :returns: A socket. + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(...) + socket = c.connect() + socket.write("POST %s HTTP/1.1\\r\\n" % "some/path/to/post/to") + socket.write("Host: %s:%s\\r\\n" % (c.host, c.port)) + socket.write("Accept-Encoding: identity\\r\\n") + socket.write("Authorization: %s\\r\\n" % c.token) + socket.write("X-Splunk-Input-Mode: Streaming\\r\\n") + socket.write("\\r\\n") + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if self.scheme == "https": + sock = ssl.wrap_socket(sock) + sock.connect((socket.gethostbyname(self.host), self.port)) + return sock + + @_authentication + @_log_duration + def delete(self, path_segment, owner=None, app=None, sharing=None, **query): + """Performs a DELETE operation at the REST path segment with the given + namespace and query. + + This method is named to match the HTTP method. ``delete`` makes at least + one round trip to the server, one additional round trip for each 303 + status returned, and at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.delete('saved/searches/boris') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '1786'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:53:06 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + c.delete('nonexistant/path') # raises HTTPError + c.logout() + c.delete('apps/local') # raises AuthenticationError + """ + path = self.authority + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) + logger.debug("DELETE request to %s (body: %s)", path, mask_sensitive_data(query)) + response = self.http.delete(path, self._auth_headers, **query) + return response + + @_authentication + @_log_duration + def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, **query): + """Performs a GET operation from the REST path segment with the given + namespace and query. + + This method is named to match the HTTP method. ``get`` makes at least + one round trip to the server, one additional round trip for each 303 + status returned, and at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.get('apps/local') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '26208'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:30:35 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + c.get('nonexistant/path') # raises HTTPError + c.logout() + c.get('apps/local') # raises AuthenticationError + """ + if headers is None: + headers = [] + + path = self.authority + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) + logger.debug("GET request to %s (body: %s)", path, mask_sensitive_data(query)) + all_headers = headers + self.additional_headers + self._auth_headers + response = self.http.get(path, all_headers, **query) + return response + + @_authentication + @_log_duration + def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, **query): + """Performs a POST operation from the REST path segment with the given + namespace and query. + + This method is named to match the HTTP method. ``post`` makes at least + one round trip to the server, one additional round trip for each 303 + status returned, and at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + Some of Splunk's endpoints, such as ``receivers/simple`` and + ``receivers/stream``, require unstructured data in the POST body + and all metadata passed as GET-style arguments. If you provide + a ``body`` argument to ``post``, it will be used as the POST + body, and all other keyword arguments will be passed as + GET-style arguments in the URL. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param query: All other keyword arguments, which are used as query + parameters. + :param body: Parameters to be used in the post body. If specified, + any parameters in the query will be applied to the URL instead of + the body. If a dict is supplied, the key-value pairs will be form + encoded. If a string is supplied, the body will be passed through + in the request unchanged. + :type body: ``dict`` or ``str`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.post('saved/searches', name='boris', + search='search * earliest=-1m | head 1') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '10455'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:46:06 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'Created', + 'status': 201} + c.post('nonexistant/path') # raises HTTPError + c.logout() + # raises AuthenticationError: + c.post('saved/searches', name='boris', + search='search * earliest=-1m | head 1') + """ + if headers is None: + headers = [] + + path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) + + logger.debug("POST request to %s (body: %s)", path, mask_sensitive_data(query)) + all_headers = headers + self.additional_headers + self._auth_headers + response = self.http.post(path, all_headers, **query) + return response + + @_authentication + @_log_duration + def request(self, path_segment, method="GET", headers=None, body={}, + owner=None, app=None, sharing=None): + """Issues an arbitrary HTTP request to the REST path segment. + + This method is named to match ``httplib.request``. This function + makes a single round trip to the server. + + If *owner*, *app*, and *sharing* are omitted, this method uses the + default :class:`Context` namespace. All other keyword arguments are + included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Context`` object is not + logged in. + :raises HTTPError: Raised when an error occurred in a GET operation from + *path_segment*. + :param path_segment: A REST path segment. + :type path_segment: ``string`` + :param method: The HTTP method to use (optional). + :type method: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param body: Content of the HTTP request (optional). + :type body: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + c = binding.connect(...) + c.request('saved/searches', method='GET') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '46722'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 17:24:19 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + c.request('nonexistant/path', method='GET') # raises HTTPError + c.logout() + c.get('apps/local') # raises AuthenticationError + """ + if headers is None: + headers = [] + + path = self.authority \ + + self._abspath(path_segment, owner=owner, + app=app, sharing=sharing) + + all_headers = headers + self.additional_headers + self._auth_headers + logger.debug("%s request to %s (headers: %s, body: %s)", + method, path, str(mask_sensitive_data(dict(all_headers))), mask_sensitive_data(body)) + if body: + body = _encode(**body) + + if method == "GET": + path = path + UrlEncoded('?' + body, skip_encode=True) + message = {'method': method, + 'headers': all_headers} + else: + message = {'method': method, + 'headers': all_headers, + 'body': body} + else: + message = {'method': method, + 'headers': all_headers} + + response = self.http.request(path, message) + + return response + + def login(self): + """Logs into the Splunk instance referred to by the :class:`Context` + object. + + Unless a ``Context`` is created with an explicit authentication token + (probably obtained by logging in from a different ``Context`` object) + you must call :meth:`login` before you can issue requests. + The authentication token obtained from the server is stored in the + ``token`` field of the ``Context`` object. + + :raises AuthenticationError: Raised when login fails. + :returns: The ``Context`` object, so you can chain calls. + + **Example**:: + + import splunklib.binding as binding + c = binding.Context(...).login() + # Then issue requests... + """ + + if self.has_cookies() and \ + (not self.username and not self.password): + # If we were passed session cookie(s), but no username or + # password, then login is a nop, since we're automatically + # logged in. + return + + if self.token is not _NoAuthenticationToken and \ + (not self.username and not self.password): + # If we were passed a session token, but no username or + # password, then login is a nop, since we're automatically + # logged in. + return + + if self.basic and (self.username and self.password): + # Basic auth mode requested, so this method is a nop as long + # as credentials were passed in. + return + + if self.bearerToken: + # Bearer auth mode requested, so this method is a nop as long + # as authentication token was passed in. + return + # Only try to get a token and updated cookie if username & password are specified + try: + response = self.http.post( + self.authority + self._abspath("/services/auth/login"), + username=self.username, + password=self.password, + headers=self.additional_headers, + cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header + + body = response.body.read() + session = XML(body).findtext("./sessionKey") + self.token = f"Splunk {session}" + return self + except HTTPError as he: + if he.status == 401: + raise AuthenticationError("Login failed.", he) + else: + raise + + def logout(self): + """Forgets the current session token, and cookies.""" + self.token = _NoAuthenticationToken + self.http._cookies = {} + return self + + def _abspath(self, path_segment, + owner=None, app=None, sharing=None): + """Qualifies *path_segment* into an absolute path for a URL. + + If *path_segment* is already absolute, returns it unchanged. + If *path_segment* is relative, then qualifies it with either + the provided namespace arguments or the ``Context``'s default + namespace. Any forbidden characters in *path_segment* are URL + encoded. This function has no network activity. + + Named to be consistent with RFC2396_. + + .. _RFC2396: http://www.ietf.org/rfc/rfc2396.txt + + :param path_segment: A relative or absolute URL path segment. + :type path_segment: ``string`` + :param owner, app, sharing: Components of a namespace (defaults + to the ``Context``'s namespace if all + three are omitted) + :type owner, app, sharing: ``string`` + :return: A ``UrlEncoded`` (a subclass of ``str``). + :rtype: ``string`` + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(owner='boris', app='search', sharing='user') + c._abspath('/a/b/c') == '/a/b/c' + c._abspath('/a/b c/d') == '/a/b%20c/d' + c._abspath('apps/local/search') == \ + '/servicesNS/boris/search/apps/local/search' + c._abspath('apps/local/search', sharing='system') == \ + '/servicesNS/nobody/system/apps/local/search' + url = c.authority + c._abspath('apps/local/sharing') + """ + skip_encode = isinstance(path_segment, UrlEncoded) + # If path_segment is absolute, escape all forbidden characters + # in it and return it. + if path_segment.startswith('/'): + return UrlEncoded(path_segment, skip_encode=skip_encode) + + # path_segment is relative, so we need a namespace to build an + # absolute path. + if owner or app or sharing: + ns = namespace(owner=owner, app=app, sharing=sharing) + else: + ns = self.namespace + + # If no app or owner are specified, then use the /services + # endpoint. Otherwise, use /servicesNS with the specified + # namespace. If only one of app and owner is specified, use + # '-' for the other. + if ns.app is None and ns.owner is None: + return UrlEncoded(f"/services/{path_segment}", skip_encode=skip_encode) + + oname = "nobody" if ns.owner is None else ns.owner + aname = "system" if ns.app is None else ns.app + path = UrlEncoded(f"/servicesNS/{oname}/{aname}/{path_segment}", skip_encode=skip_encode) + return path + + +def connect(**kwargs): + """This function returns an authenticated :class:`Context` object. + + This function is a shorthand for calling :meth:`Context.login`. + + This function makes one round trip to the server. + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param owner: The owner context of the namespace (the default is "None"). + :type owner: ``string`` + :param app: The app context of the namespace (the default is "None"). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (the default is "user"). + :type sharing: "global", "system", "app", or "user" + :param token: The current session token (optional). Session tokens can be + shared across multiple service instances. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param username: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param password: The password for the Splunk account. + :type password: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. + :param autologin: When ``True``, automatically tries to log in again if the + session terminates. + :type autologin: ``Boolean`` + :return: An initialized :class:`Context` instance. + + **Example**:: + + import splunklib.binding as binding + c = binding.connect(...) + response = c.get("apps/local") + """ + c = Context(**kwargs) + c.login() + return c + + +# Note: the error response schema supports multiple messages but we only +# return the first, although we do return the body so that an exception +# handler that wants to read multiple messages can do so. +class HTTPError(Exception): + """This exception is raised for HTTP responses that return an error.""" + + def __init__(self, response, _message=None): + status = response.status + reason = response.reason + body = response.body.read() + try: + detail = XML(body).findtext("./messages/msg") + except ParseError: + detail = body + detail_formatted = "" if detail is None else f" -- {detail}" + message = f"HTTP {status} {reason}{detail_formatted}" + Exception.__init__(self, _message or message) + self.status = status + self.reason = reason + self.headers = response.headers + self.body = body + self._response = response + + +class AuthenticationError(HTTPError): + """Raised when a login request to Splunk fails. + + If your username was unknown or you provided an incorrect password + in a call to :meth:`Context.login` or :meth:`splunklib.client.Service.login`, + this exception is raised. + """ + + def __init__(self, message, cause): + # Put the body back in the response so that HTTPError's constructor can + # read it again. + cause._response.body = BytesIO(cause.body) + + HTTPError.__init__(self, cause._response, message) + + +# +# The HTTP interface used by the Splunk binding layer abstracts the underlying +# HTTP library using request & response 'messages' which are implemented as +# dictionaries with the following structure: +# +# # HTTP request message (only method required) +# request { +# method : str, +# headers? : [(str, str)*], +# body? : str, +# } +# +# # HTTP response message (all keys present) +# response { +# status : int, +# reason : str, +# headers : [(str, str)*], +# body : file, +# } +# + +# Encode the given kwargs as a query string. This wrapper will also _encode +# a list value as a sequence of assignments to the corresponding arg name, +# for example an argument such as 'foo=[1,2,3]' will be encoded as +# 'foo=1&foo=2&foo=3'. +def _encode(**kwargs): + items = [] + for key, value in kwargs.items(): + if isinstance(value, list): + items.extend([(key, item) for item in value]) + else: + items.append((key, value)) + return parse.urlencode(items) + + +# Crack the given url into (scheme, host, port, path) +def _spliturl(url): + parsed_url = parse.urlparse(url) + host = parsed_url.hostname + port = parsed_url.port + path = '?'.join((parsed_url.path, parsed_url.query)) if parsed_url.query else parsed_url.path + # Strip brackets if its an IPv6 address + if host.startswith('[') and host.endswith(']'): host = host[1:-1] + if port is None: port = DEFAULT_PORT + return parsed_url.scheme, host, port, path + + +# Given an HTTP request handler, this wrapper objects provides a related +# family of convenience methods built using that handler. +class HttpLib: + """A set of convenient methods for making HTTP calls. + + ``HttpLib`` provides a general :meth:`request` method, and :meth:`delete`, + :meth:`post`, and :meth:`get` methods for the three HTTP methods that Splunk + uses. + + By default, ``HttpLib`` uses Python's built-in ``httplib`` library, + but you can replace it by passing your own handling function to the + constructor for ``HttpLib``. + + The handling function should have the type: + + ``handler(`url`, `request_dict`) -> response_dict`` + + where `url` is the URL to make the request to (including any query and + fragment sections) as a dictionary with the following keys: + + - method: The method for the request, typically ``GET``, ``POST``, or ``DELETE``. + + - headers: A list of pairs specifying the HTTP headers (for example: ``[('key': value), ...]``). + + - body: A string containing the body to send with the request (this string + should default to ''). + + and ``response_dict`` is a dictionary with the following keys: + + - status: An integer containing the HTTP status code (such as 200 or 404). + + - reason: The reason phrase, if any, returned by the server. + + - headers: A list of pairs containing the response headers (for example, ``[('key': value), ...]``). + + - body: A stream-like object supporting ``read(size=None)`` and ``close()`` + methods to get the body of the response. + + The response dictionary is returned directly by ``HttpLib``'s methods with + no further processing. By default, ``HttpLib`` calls the :func:`handler` function + to get a handler function. + + If using the default handler, SSL verification can be disabled by passing verify=False. + """ + + def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None, retries=0, + retryDelay=10): + if custom_handler is None: + self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file, context=context) + else: + self.handler = custom_handler + self._cookies = {} + self.retries = retries + self.retryDelay = retryDelay + + def delete(self, url, headers=None, **kwargs): + """Sends a DELETE request to a URL. + + :param url: The URL. + :type url: ``string`` + :param headers: A list of pairs specifying the headers for the HTTP + response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). + :type headers: ``list`` + :param kwargs: Additional keyword arguments (optional). These arguments + are interpreted as the query part of the URL. The order of keyword + arguments is not preserved in the request, but the keywords and + their arguments will be URL encoded. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + if headers is None: headers = [] + if kwargs: + # url is already a UrlEncoded. We have to manually declare + # the query to be encoded or it will get automatically URL + # encoded by being appended to url. + url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) + message = { + 'method': "DELETE", + 'headers': headers, + } + return self.request(url, message) + + def get(self, url, headers=None, **kwargs): + """Sends a GET request to a URL. + + :param url: The URL. + :type url: ``string`` + :param headers: A list of pairs specifying the headers for the HTTP + response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). + :type headers: ``list`` + :param kwargs: Additional keyword arguments (optional). These arguments + are interpreted as the query part of the URL. The order of keyword + arguments is not preserved in the request, but the keywords and + their arguments will be URL encoded. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + if headers is None: headers = [] + if kwargs: + # url is already a UrlEncoded. We have to manually declare + # the query to be encoded or it will get automatically URL + # encoded by being appended to url. + url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) + return self.request(url, {'method': "GET", 'headers': headers}) + + def post(self, url, headers=None, **kwargs): + """Sends a POST request to a URL. + + :param url: The URL. + :type url: ``string`` + :param headers: A list of pairs specifying the headers for the HTTP + response (for example, ``[('Content-Type': 'text/cthulhu'), ('Token': 'boris')]``). + :type headers: ``list`` + :param kwargs: Additional keyword arguments (optional). If the argument + is ``body``, the value is used as the body for the request, and the + keywords and their arguments will be URL encoded. If there is no + ``body`` keyword argument, all the keyword arguments are encoded + into the body of the request in the format ``x-www-form-urlencoded``. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + if headers is None: headers = [] + + # We handle GET-style arguments and an unstructured body. This is here + # to support the receivers/stream endpoint. + if 'body' in kwargs: + # We only use application/x-www-form-urlencoded if there is no other + # Content-Type header present. This can happen in cases where we + # send requests as application/json, e.g. for KV Store. + if len([x for x in headers if x[0].lower() == "content-type"]) == 0: + headers.append(("Content-Type", "application/x-www-form-urlencoded")) + + body = kwargs.pop('body') + if isinstance(body, dict): + body = _encode(**body).encode('utf-8') + if len(kwargs) > 0: + url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True) + else: + body = _encode(**kwargs).encode('utf-8') + message = { + 'method': "POST", + 'headers': headers, + 'body': body + } + return self.request(url, message) + + def request(self, url, message, **kwargs): + """Issues an HTTP request to a URL. + + :param url: The URL. + :type url: ``string`` + :param message: A dictionary with the format as described in + :class:`HttpLib`. + :type message: ``dict`` + :param kwargs: Additional keyword arguments (optional). These arguments + are passed unchanged to the handler. + :type kwargs: ``dict`` + :returns: A dictionary describing the response (see :class:`HttpLib` for + its structure). + :rtype: ``dict`` + """ + while True: + try: + response = self.handler(url, message, **kwargs) + break + except Exception: + if self.retries <= 0: + raise + else: + time.sleep(self.retryDelay) + self.retries -= 1 + response = record(response) + if 400 <= response.status: + raise HTTPError(response) + + # Update the cookie with any HTTP request + # Initially, assume list of 2-tuples + key_value_tuples = response.headers + # If response.headers is a dict, get the key-value pairs as 2-tuples + # this is the case when using urllib2 + if isinstance(response.headers, dict): + key_value_tuples = list(response.headers.items()) + for key, value in key_value_tuples: + if key.lower() == "set-cookie": + _parse_cookies(value, self._cookies) + + return response + + +# Converts an httplib response into a file-like object. +class ResponseReader(io.RawIOBase): + """This class provides a file-like interface for :class:`httplib` responses. + + The ``ResponseReader`` class is intended to be a layer to unify the different + types of HTTP libraries used with this SDK. This class also provides a + preview of the stream and a few useful predicates. + """ + + # For testing, you can use a StringIO as the argument to + # ``ResponseReader`` instead of an ``httplib.HTTPResponse``. It + # will work equally well. + def __init__(self, response, connection=None): + self._response = response + self._connection = connection + self._buffer = b'' + + def __str__(self): + return str(self.read(), 'UTF-8') + + @property + def empty(self): + """Indicates whether there is any more data in the response.""" + return self.peek(1) == b"" + + def peek(self, size): + """Nondestructively retrieves a given number of characters. + + The next :meth:`read` operation behaves as though this method was never + called. + + :param size: The number of characters to retrieve. + :type size: ``integer`` + """ + c = self.read(size) + self._buffer = self._buffer + c + return c + + def close(self): + """Closes this response.""" + if self._connection: + self._connection.close() + self._response.close() + + def read(self, size=None): + """Reads a given number of characters from the response. + + :param size: The number of characters to read, or "None" to read the + entire response. + :type size: ``integer`` or "None" + + """ + r = self._buffer + self._buffer = b'' + if size is not None: + size -= len(r) + r = r + self._response.read(size) + return r + + def readable(self): + """ Indicates that the response reader is readable.""" + return True + + def readinto(self, byte_array): + """ Read data into a byte array, upto the size of the byte array. + + :param byte_array: A byte array/memory view to pour bytes into. + :type byte_array: ``bytearray`` or ``memoryview`` + + """ + max_size = len(byte_array) + data = self.read(max_size) + bytes_read = len(data) + byte_array[:bytes_read] = data + return bytes_read + + +def handler(key_file=None, cert_file=None, timeout=None, verify=False, context=None): + """This class returns an instance of the default HTTP request handler using + the values you provide. + + :param `key_file`: A path to a PEM (Privacy Enhanced Mail) formatted file containing your private key (optional). + :type key_file: ``string`` + :param `cert_file`: A path to a PEM (Privacy Enhanced Mail) formatted file containing a certificate chain file (optional). + :type cert_file: ``string`` + :param `timeout`: The request time-out period, in seconds (optional). + :type timeout: ``integer`` or "None" + :param `verify`: Set to False to disable SSL verification on https connections. + :type verify: ``Boolean`` + :param `context`: The SSLContext that can is used with the HTTPSConnection when verify=True is enabled and context is specified + :type context: ``SSLContext` + """ + + def connect(scheme, host, port): + kwargs = {} + if timeout is not None: kwargs['timeout'] = timeout + if scheme == "http": + return client.HTTPConnection(host, port, **kwargs) + if scheme == "https": + if key_file is not None: kwargs['key_file'] = key_file + if cert_file is not None: kwargs['cert_file'] = cert_file + + if not verify: + kwargs['context'] = ssl._create_unverified_context() + elif context: + # verify is True in elif branch and context is not None + kwargs['context'] = context + + return client.HTTPSConnection(host, port, **kwargs) + raise ValueError(f"unsupported scheme: {scheme}") + + def request(url, message, **kwargs): + scheme, host, port, path = _spliturl(url) + body = message.get("body", "") + head = { + "Content-Length": str(len(body)), + "Host": host, + "User-Agent": "splunk-sdk-python/%s" % __version__, + "Accept": "*/*", + "Connection": "Close", + } # defaults + for key, value in message["headers"]: + head[key] = value + method = message.get("method", "GET") + + connection = connect(scheme, host, port) + is_keepalive = False + try: + connection.request(method, path, body, head) + if timeout is not None: + connection.sock.settimeout(timeout) + response = connection.getresponse() + is_keepalive = "keep-alive" in response.getheader("connection", default="close").lower() + finally: + if not is_keepalive: + connection.close() + + return { + "status": response.status, + "reason": response.reason, + "headers": response.getheaders(), + "body": ResponseReader(response, connection if is_keepalive else None), + } + + return request diff --git a/splunklib/client.py b/splunklib/client.py index 090f9192..48861880 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -1,3908 +1,3908 @@ -# Copyright © 2011-2024 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# The purpose of this module is to provide a friendlier domain interface to -# various Splunk endpoints. The approach here is to leverage the binding -# layer to capture endpoint context and provide objects and methods that -# offer simplified access their corresponding endpoints. The design avoids -# caching resource state. From the perspective of this module, the 'policy' -# for caching resource state belongs in the application or a higher level -# framework, and its the purpose of this module to provide simplified -# access to that resource state. -# -# A side note, the objects below that provide helper methods for updating eg: -# Entity state, are written so that they may be used in a fluent style. -# - -"""The **splunklib.client** module provides a Pythonic interface to the -`Splunk REST API `_, -allowing you programmatically access Splunk's resources. - -**splunklib.client** wraps a Pythonic layer around the wire-level -binding of the **splunklib.binding** module. The core of the library is the -:class:`Service` class, which encapsulates a connection to the server, and -provides access to the various aspects of Splunk's functionality, which are -exposed via the REST API. Typically you connect to a running Splunk instance -with the :func:`connect` function:: - - import splunklib.client as client - service = client.connect(host='localhost', port=8089, - username='admin', password='...') - assert isinstance(service, client.Service) - -:class:`Service` objects have fields for the various Splunk resources (such as apps, -jobs, saved searches, inputs, and indexes). All of these fields are -:class:`Collection` objects:: - - appcollection = service.apps - my_app = appcollection.create('my_app') - my_app = appcollection['my_app'] - appcollection.delete('my_app') - -The individual elements of the collection, in this case *applications*, -are subclasses of :class:`Entity`. An ``Entity`` object has fields for its -attributes, and methods that are specific to each kind of entity. For example:: - - print(my_app['author']) # Or: print(my_app.author) - my_app.package() # Creates a compressed package of this application -""" - -import contextlib -import datetime -import json -import logging -import re -import socket -from datetime import datetime, timedelta -from time import sleep -from urllib import parse - -from splunklib import data -from splunklib.data import record -from splunklib.binding import (AuthenticationError, Context, HTTPError, UrlEncoded, - _encode, _make_cookie_header, _NoAuthenticationToken, - namespace) - -logger = logging.getLogger(__name__) - -__all__ = [ - "connect", - "NotSupportedError", - "OperationError", - "IncomparableException", - "Service", - "namespace", - "AuthenticationError" -] - -PATH_APPS = "apps/local/" -PATH_CAPABILITIES = "authorization/capabilities/" -PATH_CONF = "configs/conf-%s/" -PATH_PROPERTIES = "properties/" -PATH_DEPLOYMENT_CLIENTS = "deployment/client/" -PATH_DEPLOYMENT_TENANTS = "deployment/tenants/" -PATH_DEPLOYMENT_SERVERS = "deployment/server/" -PATH_DEPLOYMENT_SERVERCLASSES = "deployment/serverclass/" -PATH_EVENT_TYPES = "saved/eventtypes/" -PATH_FIRED_ALERTS = "alerts/fired_alerts/" -PATH_INDEXES = "data/indexes/" -PATH_INPUTS = "data/inputs/" -PATH_JOBS = "search/jobs/" -PATH_JOBS_V2 = "search/v2/jobs/" -PATH_LOGGER = "/services/server/logger/" -PATH_MESSAGES = "messages/" -PATH_MODULAR_INPUTS = "data/modular-inputs" -PATH_ROLES = "authorization/roles/" -PATH_SAVED_SEARCHES = "saved/searches/" -PATH_STANZA = "configs/conf-%s/%s" # (file, stanza) -PATH_USERS = "authentication/users/" -PATH_RECEIVERS_STREAM = "/services/receivers/stream" -PATH_RECEIVERS_SIMPLE = "/services/receivers/simple" -PATH_STORAGE_PASSWORDS = "storage/passwords" - -XNAMEF_ATOM = "{http://www.w3.org/2005/Atom}%s" -XNAME_ENTRY = XNAMEF_ATOM % "entry" -XNAME_CONTENT = XNAMEF_ATOM % "content" - -MATCH_ENTRY_CONTENT = f"{XNAME_ENTRY}/{XNAME_CONTENT}/*" - - -class IllegalOperationException(Exception): - """Thrown when an operation is not possible on the Splunk instance that a - :class:`Service` object is connected to.""" - - -class IncomparableException(Exception): - """Thrown when trying to compare objects (using ``==``, ``<``, ``>``, and - so on) of a type that doesn't support it.""" - - -class AmbiguousReferenceException(ValueError): - """Thrown when the name used to fetch an entity matches more than one entity.""" - - -class InvalidNameException(Exception): - """Thrown when the specified name contains characters that are not allowed - in Splunk entity names.""" - - -class NoSuchCapability(Exception): - """Thrown when the capability that has been referred to doesn't exist.""" - - -class OperationError(Exception): - """Raised for a failed operation, such as a timeout.""" - - -class NotSupportedError(Exception): - """Raised for operations that are not supported on a given object.""" - - -def _trailing(template, *targets): - """Substring of *template* following all *targets*. - - **Example**:: - - template = "this is a test of the bunnies." - _trailing(template, "is", "est", "the") == " bunnies" - - Each target is matched successively in the string, and the string - remaining after the last target is returned. If one of the targets - fails to match, a ValueError is raised. - - :param template: Template to extract a trailing string from. - :type template: ``string`` - :param targets: Strings to successively match in *template*. - :type targets: list of ``string``s - :return: Trailing string after all targets are matched. - :rtype: ``string`` - :raises ValueError: Raised when one of the targets does not match. - """ - s = template - for t in targets: - n = s.find(t) - if n == -1: - raise ValueError("Target " + t + " not found in template.") - s = s[n + len(t):] - return s - - -# Filter the given state content record according to the given arg list. -def _filter_content(content, *args): - if len(args) > 0: - return record((k, content[k]) for k in args) - return record((k, v) for k, v in content.items() - if k not in ['eai:acl', 'eai:attributes', 'type']) - - -# Construct a resource path from the given base path + resource name -def _path(base, name): - if not base.endswith('/'): base = base + '/' - return base + name - - -# Load an atom record from the body of the given response -# this will ultimately be sent to an xml ElementTree so we -# should use the xmlcharrefreplace option -def _load_atom(response, match=None): - return data.load(response.body.read() - .decode('utf-8', 'xmlcharrefreplace'), match) - - -# Load an array of atom entries from the body of the given response -def _load_atom_entries(response): - r = _load_atom(response) - if 'feed' in r: - # Need this to handle a random case in the REST API - if r.feed.get('totalResults') in [0, '0']: - return [] - entries = r.feed.get('entry', None) - if entries is None: return None - return entries if isinstance(entries, list) else [entries] - # Unlike most other endpoints, the jobs endpoint does not return - # its state wrapped in another element, but at the top level. - # For example, in XML, it returns ... instead of - # .... - entries = r.get('entry', None) - if entries is None: return None - return entries if isinstance(entries, list) else [entries] - - -# Load the sid from the body of the given response -def _load_sid(response, output_mode): - if output_mode == "json": - json_obj = json.loads(response.body.read()) - return json_obj.get('sid') - return _load_atom(response).response.sid - - -# Parse the given atom entry record into a generic entity state record -def _parse_atom_entry(entry): - title = entry.get('title', None) - - elink = entry.get('link', []) - elink = elink if isinstance(elink, list) else [elink] - links = record((link.rel, link.href) for link in elink) - - # Retrieve entity content values - content = entry.get('content', {}) - - # Host entry metadata - metadata = _parse_atom_metadata(content) - - # Filter some of the noise out of the content record - content = record((k, v) for k, v in content.items() - if k not in ['eai:acl', 'eai:attributes']) - - if 'type' in content: - if isinstance(content['type'], list): - content['type'] = [t for t in content['type'] if t != 'text/xml'] - # Unset type if it was only 'text/xml' - if len(content['type']) == 0: - content.pop('type', None) - # Flatten 1 element list - if len(content['type']) == 1: - content['type'] = content['type'][0] - else: - content.pop('type', None) - - return record({ - 'title': title, - 'links': links, - 'access': metadata.access, - 'fields': metadata.fields, - 'content': content, - 'updated': entry.get("updated") - }) - - -# Parse the metadata fields out of the given atom entry content record -def _parse_atom_metadata(content): - # Hoist access metadata - access = content.get('eai:acl', None) - - # Hoist content metadata (and cleanup some naming) - attributes = content.get('eai:attributes', {}) - fields = record({ - 'required': attributes.get('requiredFields', []), - 'optional': attributes.get('optionalFields', []), - 'wildcard': attributes.get('wildcardFields', [])}) - - return record({'access': access, 'fields': fields}) - - -# kwargs: scheme, host, port, app, owner, username, password -def connect(**kwargs): - """This function connects and logs in to a Splunk instance. - - This function is a shorthand for :meth:`Service.login`. - The ``connect`` function makes one round trip to the server (for logging in). - - :param host: The host name (the default is "localhost"). - :type host: ``string`` - :param port: The port number (the default is 8089). - :type port: ``integer`` - :param scheme: The scheme for accessing the service (the default is "https"). - :type scheme: "https" or "http" - :param verify: Enable (True) or disable (False) SSL verification for - https connections. (optional, the default is True) - :type verify: ``Boolean`` - :param `owner`: The owner context of the namespace (optional). - :type owner: ``string`` - :param `app`: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode for the namespace (the default is "user"). - :type sharing: "global", "system", "app", or "user" - :param `token`: The current session token (optional). Session tokens can be - shared across multiple service instances. - :type token: ``string`` - :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. - This parameter is only supported for Splunk 6.2+. - :type cookie: ``string`` - :param autologin: When ``True``, automatically tries to log in again if the - session terminates. - :type autologin: ``boolean`` - :param `username`: The Splunk account username, which is used to - authenticate the Splunk instance. - :type username: ``string`` - :param `password`: The password for the Splunk account. - :type password: ``string`` - :param retires: Number of retries for each HTTP connection (optional, the default is 0). - NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. - :type retries: ``int`` - :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). - :type retryDelay: ``int`` (in seconds) - :param `context`: The SSLContext that can be used when setting verify=True (optional) - :type context: ``SSLContext`` - :return: An initialized :class:`Service` connection. - - **Example**:: - - import splunklib.client as client - s = client.connect(...) - a = s.apps["my_app"] - ... - """ - s = Service(**kwargs) - s.login() - return s - - -# In preparation for adding Storm support, we added an -# intermediary class between Service and Context. Storm's -# API is not going to be the same as enterprise Splunk's -# API, so we will derive both Service (for enterprise Splunk) -# and StormService for (Splunk Storm) from _BaseService, and -# put any shared behavior on it. -class _BaseService(Context): - pass - - -class Service(_BaseService): - """A Pythonic binding to Splunk instances. - - A :class:`Service` represents a binding to a Splunk instance on an - HTTP or HTTPS port. It handles the details of authentication, wire - formats, and wraps the REST API endpoints into something more - Pythonic. All of the low-level operations on the instance from - :class:`splunklib.binding.Context` are also available in case you need - to do something beyond what is provided by this class. - - After creating a ``Service`` object, you must call its :meth:`login` - method before you can issue requests to Splunk. - Alternately, use the :func:`connect` function to create an already - authenticated :class:`Service` object, or provide a session token - when creating the :class:`Service` object explicitly (the same - token may be shared by multiple :class:`Service` objects). - - :param host: The host name (the default is "localhost"). - :type host: ``string`` - :param port: The port number (the default is 8089). - :type port: ``integer`` - :param scheme: The scheme for accessing the service (the default is "https"). - :type scheme: "https" or "http" - :param verify: Enable (True) or disable (False) SSL verification for - https connections. (optional, the default is True) - :type verify: ``Boolean`` - :param `owner`: The owner context of the namespace (optional; use "-" for wildcard). - :type owner: ``string`` - :param `app`: The app context of the namespace (optional; use "-" for wildcard). - :type app: ``string`` - :param `token`: The current session token (optional). Session tokens can be - shared across multiple service instances. - :type token: ``string`` - :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. - This parameter is only supported for Splunk 6.2+. - :type cookie: ``string`` - :param `username`: The Splunk account username, which is used to - authenticate the Splunk instance. - :type username: ``string`` - :param `password`: The password, which is used to authenticate the Splunk - instance. - :type password: ``string`` - :param retires: Number of retries for each HTTP connection (optional, the default is 0). - NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. - :type retries: ``int`` - :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). - :type retryDelay: ``int`` (in seconds) - :return: A :class:`Service` instance. - - **Example**:: - - import splunklib.client as client - s = client.Service(username="boris", password="natasha", ...) - s.login() - # Or equivalently - s = client.connect(username="boris", password="natasha") - # Or if you already have a session token - s = client.Service(token="atg232342aa34324a") - # Or if you already have a valid cookie - s = client.Service(cookie="splunkd_8089=...") - """ - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self._splunk_version = None - self._kvstore_owner = None - self._instance_type = None - - @property - def apps(self): - """Returns the collection of applications that are installed on this instance of Splunk. - - :return: A :class:`Collection` of :class:`Application` entities. - """ - return Collection(self, PATH_APPS, item=Application) - - @property - def confs(self): - """Returns the collection of configuration files for this Splunk instance. - - :return: A :class:`Configurations` collection of - :class:`ConfigurationFile` entities. - """ - return Configurations(self) - - @property - def capabilities(self): - """Returns the list of system capabilities. - - :return: A ``list`` of capabilities. - """ - response = self.get(PATH_CAPABILITIES) - return _load_atom(response, MATCH_ENTRY_CONTENT).capabilities - - @property - def event_types(self): - """Returns the collection of event types defined in this Splunk instance. - - :return: An :class:`Entity` containing the event types. - """ - return Collection(self, PATH_EVENT_TYPES) - - @property - def fired_alerts(self): - """Returns the collection of alerts that have been fired on the Splunk - instance, grouped by saved search. - - :return: A :class:`Collection` of :class:`AlertGroup` entities. - """ - return Collection(self, PATH_FIRED_ALERTS, item=AlertGroup) - - @property - def indexes(self): - """Returns the collection of indexes for this Splunk instance. - - :return: An :class:`Indexes` collection of :class:`Index` entities. - """ - return Indexes(self, PATH_INDEXES, item=Index) - - @property - def info(self): - """Returns the information about this instance of Splunk. - - :return: The system information, as key-value pairs. - :rtype: ``dict`` - """ - response = self.get("/services/server/info") - return _filter_content(_load_atom(response, MATCH_ENTRY_CONTENT)) - - def input(self, path, kind=None): - """Retrieves an input by path, and optionally kind. - - :return: A :class:`Input` object. - """ - return Input(self, path, kind=kind).refresh() - - @property - def inputs(self): - """Returns the collection of inputs configured on this Splunk instance. - - :return: An :class:`Inputs` collection of :class:`Input` entities. - """ - return Inputs(self) - - def job(self, sid): - """Retrieves a search job by sid. - - :return: A :class:`Job` object. - """ - return Job(self, sid).refresh() - - @property - def jobs(self): - """Returns the collection of current search jobs. - - :return: A :class:`Jobs` collection of :class:`Job` entities. - """ - return Jobs(self) - - @property - def loggers(self): - """Returns the collection of logging level categories and their status. - - :return: A :class:`Loggers` collection of logging levels. - """ - return Loggers(self) - - @property - def messages(self): - """Returns the collection of service messages. - - :return: A :class:`Collection` of :class:`Message` entities. - """ - return Collection(self, PATH_MESSAGES, item=Message) - - @property - def modular_input_kinds(self): - """Returns the collection of the modular input kinds on this Splunk instance. - - :return: A :class:`ReadOnlyCollection` of :class:`ModularInputKind` entities. - """ - if self.splunk_version >= (5,): - return ReadOnlyCollection(self, PATH_MODULAR_INPUTS, item=ModularInputKind) - raise IllegalOperationException("Modular inputs are not supported before Splunk version 5.") - - @property - def storage_passwords(self): - """Returns the collection of the storage passwords on this Splunk instance. - - :return: A :class:`ReadOnlyCollection` of :class:`StoragePasswords` entities. - """ - return StoragePasswords(self) - - # kwargs: enable_lookups, reload_macros, parse_only, output_mode - def parse(self, query, **kwargs): - """Parses a search query and returns a semantic map of the search. - - :param query: The search query to parse. - :type query: ``string`` - :param kwargs: Arguments to pass to the ``search/parser`` endpoint - (optional). Valid arguments are: - - * "enable_lookups" (``boolean``): If ``True``, performs reverse lookups - to expand the search expression. - - * "output_mode" (``string``): The output format (XML or JSON). - - * "parse_only" (``boolean``): If ``True``, disables the expansion of - search due to evaluation of subsearches, time term expansion, - lookups, tags, eventtypes, and sourcetype alias. - - * "reload_macros" (``boolean``): If ``True``, reloads macro - definitions from macros.conf. - - :type kwargs: ``dict`` - :return: A semantic map of the parsed search query. - """ - if not self.disable_v2_api: - return self.post("search/v2/parser", q=query, **kwargs) - return self.get("search/parser", q=query, **kwargs) - - def restart(self, timeout=None): - """Restarts this Splunk instance. - - The service is unavailable until it has successfully restarted. - - If a *timeout* value is specified, ``restart`` blocks until the service - resumes or the timeout period has been exceeded. Otherwise, ``restart`` returns - immediately. - - :param timeout: A timeout period, in seconds. - :type timeout: ``integer`` - """ - msg = {"value": "Restart requested by " + self.username + "via the Splunk SDK for Python"} - # This message will be deleted once the server actually restarts. - self.messages.create(name="restart_required", **msg) - result = self.post("/services/server/control/restart") - if timeout is None: - return result - start = datetime.now() - diff = timedelta(seconds=timeout) - while datetime.now() - start < diff: - try: - self.login() - if not self.restart_required: - return result - except Exception as e: - sleep(1) - raise Exception("Operation time out.") - - @property - def restart_required(self): - """Indicates whether splunkd is in a state that requires a restart. - - :return: A ``boolean`` that indicates whether a restart is required. - - """ - response = self.get("messages").body.read() - messages = data.load(response)['feed'] - if 'entry' not in messages: - result = False - else: - if isinstance(messages['entry'], dict): - titles = [messages['entry']['title']] - else: - titles = [x['title'] for x in messages['entry']] - result = 'restart_required' in titles - return result - - @property - def roles(self): - """Returns the collection of user roles. - - :return: A :class:`Roles` collection of :class:`Role` entities. - """ - return Roles(self) - - def search(self, query, **kwargs): - """Runs a search using a search query and any optional arguments you - provide, and returns a `Job` object representing the search. - - :param query: A search query. - :type query: ``string`` - :param kwargs: Arguments for the search (optional): - - * "output_mode" (``string``): Specifies the output format of the - results. - - * "earliest_time" (``string``): Specifies the earliest time in the - time range to - search. The time string can be a UTC time (with fractional - seconds), a relative time specifier (to now), or a formatted - time string. - - * "latest_time" (``string``): Specifies the latest time in the time - range to - search. The time string can be a UTC time (with fractional - seconds), a relative time specifier (to now), or a formatted - time string. - - * "rf" (``string``): Specifies one or more fields to add to the - search. - - :type kwargs: ``dict`` - :rtype: class:`Job` - :returns: An object representing the created job. - """ - return self.jobs.create(query, **kwargs) - - @property - def saved_searches(self): - """Returns the collection of saved searches. - - :return: A :class:`SavedSearches` collection of :class:`SavedSearch` - entities. - """ - return SavedSearches(self) - - @property - def settings(self): - """Returns the configuration settings for this instance of Splunk. - - :return: A :class:`Settings` object containing configuration settings. - """ - return Settings(self) - - @property - def splunk_version(self): - """Returns the version of the splunkd instance this object is attached - to. - - The version is returned as a tuple of the version components as - integers (for example, `(4,3,3)` or `(5,)`). - - :return: A ``tuple`` of ``integers``. - """ - if self._splunk_version is None: - self._splunk_version = tuple(int(p) for p in self.info['version'].split('.')) - return self._splunk_version - - @property - def splunk_instance(self): - if self._instance_type is None : - splunk_info = self.info - if hasattr(splunk_info, 'instance_type') : - self._instance_type = splunk_info['instance_type'] - else: - self._instance_type = '' - return self._instance_type - - @property - def disable_v2_api(self): - if self.splunk_instance.lower() == 'cloud': - return self.splunk_version < (9,0,2209) - return self.splunk_version < (9,0,2) - - @property - def kvstore_owner(self): - """Returns the KVStore owner for this instance of Splunk. - - By default is the kvstore owner is not set, it will return "nobody" - :return: A string with the KVStore owner. - """ - if self._kvstore_owner is None: - self._kvstore_owner = "nobody" - return self._kvstore_owner - - @kvstore_owner.setter - def kvstore_owner(self, value): - """ - kvstore is refreshed, when the owner value is changed - """ - self._kvstore_owner = value - self.kvstore - - @property - def kvstore(self): - """Returns the collection of KV Store collections. - - sets the owner for the namespace, before retrieving the KVStore Collection - - :return: A :class:`KVStoreCollections` collection of :class:`KVStoreCollection` entities. - """ - self.namespace['owner'] = self.kvstore_owner - return KVStoreCollections(self) - - @property - def users(self): - """Returns the collection of users. - - :return: A :class:`Users` collection of :class:`User` entities. - """ - return Users(self) - - -class Endpoint: - """This class represents individual Splunk resources in the Splunk REST API. - - An ``Endpoint`` object represents a URI, such as ``/services/saved/searches``. - This class provides the common functionality of :class:`Collection` and - :class:`Entity` (essentially HTTP GET and POST methods). - """ - - def __init__(self, service, path): - self.service = service - self.path = path - - def get_api_version(self, path): - """Return the API version of the service used in the provided path. - - Args: - path (str): A fully-qualified endpoint path (for example, "/services/search/jobs"). - - Returns: - int: Version of the API (for example, 1) - """ - # Default to v1 if undefined in the path - # For example, "/services/search/jobs" is using API v1 - api_version = 1 - - versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) - if versionSearch: - api_version = int(versionSearch.group(1)) - - return api_version - - def get(self, path_segment="", owner=None, app=None, sharing=None, **query): - """Performs a GET operation on the path segment relative to this endpoint. - - This method is named to match the HTTP method. This method makes at least - one roundtrip to the server, one additional round trip for - each 303 status returned, plus at most two additional round - trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - If *owner*, *app*, and *sharing* are omitted, this method takes a - default namespace from the :class:`Service` object for this :class:`Endpoint`. - All other keyword arguments are included in the URL as query parameters. - - :raises AuthenticationError: Raised when the ``Service`` is not logged in. - :raises HTTPError: Raised when an error in the request occurs. - :param path_segment: A path segment relative to this endpoint. - :type path_segment: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode for the namespace (optional). - :type sharing: "global", "system", "app", or "user" - :param query: All other keyword arguments, which are used as query - parameters. - :type query: ``string`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - import splunklib.client - s = client.service(...) - apps = s.apps - apps.get() == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '26208'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 16:30:35 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'OK', - 'status': 200} - apps.get('nonexistant/path') # raises HTTPError - s.logout() - apps.get() # raises AuthenticationError - """ - # self.path to the Endpoint is relative in the SDK, so passing - # owner, app, sharing, etc. along will produce the correct - # namespace in the final request. - if path_segment.startswith('/'): - path = path_segment - else: - if not self.path.endswith('/') and path_segment != "": - self.path = self.path + '/' - path = self.service._abspath(self.path + path_segment, owner=owner, - app=app, sharing=sharing) - # ^-- This was "%s%s" % (self.path, path_segment). - # That doesn't work, because self.path may be UrlEncoded. - - # Get the API version from the path - api_version = self.get_api_version(path) - - # Search API v2+ fallback to v1: - # - In v2+, /results_preview, /events and /results do not support search params. - # - Fallback from v2+ to v1 if Splunk Version is < 9. - # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): - # path = path.replace(PATH_JOBS_V2, PATH_JOBS) - - if api_version == 1: - if isinstance(path, UrlEncoded): - path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) - else: - path = path.replace(PATH_JOBS_V2, PATH_JOBS) - - return self.service.get(path, - owner=owner, app=app, sharing=sharing, - **query) - - def post(self, path_segment="", owner=None, app=None, sharing=None, **query): - """Performs a POST operation on the path segment relative to this endpoint. - - This method is named to match the HTTP method. This method makes at least - one roundtrip to the server, one additional round trip for - each 303 status returned, plus at most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - If *owner*, *app*, and *sharing* are omitted, this method takes a - default namespace from the :class:`Service` object for this :class:`Endpoint`. - All other keyword arguments are included in the URL as query parameters. - - :raises AuthenticationError: Raised when the ``Service`` is not logged in. - :raises HTTPError: Raised when an error in the request occurs. - :param path_segment: A path segment relative to this endpoint. - :type path_segment: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode of the namespace (optional). - :type sharing: ``string`` - :param query: All other keyword arguments, which are used as query - parameters. - :type query: ``string`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - import splunklib.client - s = client.service(...) - apps = s.apps - apps.post(name='boris') == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '2908'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 18:34:50 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'Created', - 'status': 201} - apps.get('nonexistant/path') # raises HTTPError - s.logout() - apps.get() # raises AuthenticationError - """ - if path_segment.startswith('/'): - path = path_segment - else: - if not self.path.endswith('/') and path_segment != "": - self.path = self.path + '/' - path = self.service._abspath(self.path + path_segment, owner=owner, app=app, sharing=sharing) - - # Get the API version from the path - api_version = self.get_api_version(path) - - # Search API v2+ fallback to v1: - # - In v2+, /results_preview, /events and /results do not support search params. - # - Fallback from v2+ to v1 if Splunk Version is < 9. - # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): - # path = path.replace(PATH_JOBS_V2, PATH_JOBS) - - if api_version == 1: - if isinstance(path, UrlEncoded): - path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) - else: - path = path.replace(PATH_JOBS_V2, PATH_JOBS) - - return self.service.post(path, owner=owner, app=app, sharing=sharing, **query) - - -# kwargs: path, app, owner, sharing, state -class Entity(Endpoint): - """This class is a base class for Splunk entities in the REST API, such as - saved searches, jobs, indexes, and inputs. - - ``Entity`` provides the majority of functionality required by entities. - Subclasses only implement the special cases for individual entities. - For example for saved searches, the subclass makes fields like ``action.email``, - ``alert_type``, and ``search`` available. - - An ``Entity`` is addressed like a dictionary, with a few extensions, - so the following all work, for example in saved searches:: - - ent['action.email'] - ent['alert_type'] - ent['search'] - - You can also access the fields as though they were the fields of a Python - object, as in:: - - ent.alert_type - ent.search - - However, because some of the field names are not valid Python identifiers, - the dictionary-like syntax is preferable. - - The state of an :class:`Entity` object is cached, so accessing a field - does not contact the server. If you think the values on the - server have changed, call the :meth:`Entity.refresh` method. - """ - # Not every endpoint in the API is an Entity or a Collection. For - # example, a saved search at saved/searches/{name} has an additional - # method saved/searches/{name}/scheduled_times, but this isn't an - # entity in its own right. In these cases, subclasses should - # implement a method that uses the get and post methods inherited - # from Endpoint, calls the _load_atom function (it's elsewhere in - # client.py, but not a method of any object) to read the - # information, and returns the extracted data in a Pythonesque form. - # - # The primary use of subclasses of Entity is to handle specially - # named fields in the Entity. If you only need to provide a default - # value for an optional field, subclass Entity and define a - # dictionary ``defaults``. For instance,:: - # - # class Hypothetical(Entity): - # defaults = {'anOptionalField': 'foo', - # 'anotherField': 'bar'} - # - # If you have to do more than provide a default, such as rename or - # actually process values, then define a new method with the - # ``@property`` decorator. - # - # class Hypothetical(Entity): - # @property - # def foobar(self): - # return self.content['foo'] + "-" + self.content["bar"] - - # Subclasses can override defaults the default values for - # optional fields. See above. - defaults = {} - - def __init__(self, service, path, **kwargs): - Endpoint.__init__(self, service, path) - self._state = None - if not kwargs.get('skip_refresh', False): - self.refresh(kwargs.get('state', None)) # "Prefresh" - - def __contains__(self, item): - try: - self[item] - return True - except (KeyError, AttributeError): - return False - - def __eq__(self, other): - """Raises IncomparableException. - - Since Entity objects are snapshots of times on the server, no - simple definition of equality will suffice beyond instance - equality, and instance equality leads to strange situations - such as:: - - import splunklib.client as client - c = client.connect(...) - saved_searches = c.saved_searches - x = saved_searches['asearch'] - - but then ``x != saved_searches['asearch']``. - - whether or not there was a change on the server. Rather than - try to do something fancy, we simply declare that equality is - undefined for Entities. - - Makes no roundtrips to the server. - """ - raise IncomparableException(f"Equality is undefined for objects of class {self.__class__.__name__}") - - def __getattr__(self, key): - # Called when an attribute was not found by the normal method. In this - # case we try to find it in self.content and then self.defaults. - if key in self.state.content: - return self.state.content[key] - if key in self.defaults: - return self.defaults[key] - raise AttributeError(key) - - def __getitem__(self, key): - # getattr attempts to find a field on the object in the normal way, - # then calls __getattr__ if it cannot. - return getattr(self, key) - - # Load the Atom entry record from the given response - this is a method - # because the "entry" record varies slightly by entity and this allows - # for a subclass to override and handle any special cases. - def _load_atom_entry(self, response): - elem = _load_atom(response, XNAME_ENTRY) - if isinstance(elem, list): - apps = [ele.entry.content.get('eai:appName') for ele in elem] - - raise AmbiguousReferenceException( - f"Fetch from server returned multiple entries for name '{elem[0].entry.title}' in apps {apps}.") - return elem.entry - - # Load the entity state record from the given response - def _load_state(self, response): - entry = self._load_atom_entry(response) - return _parse_atom_entry(entry) - - def _run_action(self, path_segment, **kwargs): - """Run a method and return the content Record from the returned XML. - - A method is a relative path from an Entity that is not itself - an Entity. _run_action assumes that the returned XML is an - Atom field containing one Entry, and the contents of Entry is - what should be the return value. This is right in enough cases - to make this method useful. - """ - response = self.get(path_segment, **kwargs) - data = self._load_atom_entry(response) - rec = _parse_atom_entry(data) - return rec.content - - def _proper_namespace(self, owner=None, app=None, sharing=None): - """Produce a namespace sans wildcards for use in entity requests. - - This method tries to fill in the fields of the namespace which are `None` - or wildcard (`'-'`) from the entity's namespace. If that fails, it uses - the service's namespace. - - :param owner: - :param app: - :param sharing: - :return: - """ - if owner is None and app is None and sharing is None: # No namespace provided - if self._state is not None and 'access' in self._state: - return (self._state.access.owner, - self._state.access.app, - self._state.access.sharing) - return (self.service.namespace['owner'], - self.service.namespace['app'], - self.service.namespace['sharing']) - return owner, app, sharing - - def delete(self): - owner, app, sharing = self._proper_namespace() - return self.service.delete(self.path, owner=owner, app=app, sharing=sharing) - - def get(self, path_segment="", owner=None, app=None, sharing=None, **query): - owner, app, sharing = self._proper_namespace(owner, app, sharing) - return super().get(path_segment, owner=owner, app=app, sharing=sharing, **query) - - def post(self, path_segment="", owner=None, app=None, sharing=None, **query): - owner, app, sharing = self._proper_namespace(owner, app, sharing) - return super().post(path_segment, owner=owner, app=app, sharing=sharing, **query) - - def refresh(self, state=None): - """Refreshes the state of this entity. - - If *state* is provided, load it as the new state for this - entity. Otherwise, make a roundtrip to the server (by calling - the :meth:`read` method of ``self``) to fetch an updated state, - plus at most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param state: Entity-specific arguments (optional). - :type state: ``dict`` - :raises EntityDeletedException: Raised if the entity no longer exists on - the server. - - **Example**:: - - import splunklib.client as client - s = client.connect(...) - search = s.apps['search'] - search.refresh() - """ - if state is not None: - self._state = state - else: - self._state = self.read(self.get()) - return self - - @property - def access(self): - """Returns the access metadata for this entity. - - :return: A :class:`splunklib.data.Record` object with three keys: - ``owner``, ``app``, and ``sharing``. - """ - return self.state.access - - @property - def content(self): - """Returns the contents of the entity. - - :return: A ``dict`` containing values. - """ - return self.state.content - - def disable(self): - """Disables the entity at this endpoint.""" - self.post("disable") - return self - - def enable(self): - """Enables the entity at this endpoint.""" - self.post("enable") - return self - - @property - def fields(self): - """Returns the content metadata for this entity. - - :return: A :class:`splunklib.data.Record` object with three keys: - ``required``, ``optional``, and ``wildcard``. - """ - return self.state.fields - - @property - def links(self): - """Returns a dictionary of related resources. - - :return: A ``dict`` with keys and corresponding URLs. - """ - return self.state.links - - @property - def name(self): - """Returns the entity name. - - :return: The entity name. - :rtype: ``string`` - """ - return self.state.title - - def read(self, response): - """ Reads the current state of the entity from the server. """ - results = self._load_state(response) - # In lower layers of the SDK, we end up trying to URL encode - # text to be dispatched via HTTP. However, these links are already - # URL encoded when they arrive, and we need to mark them as such. - unquoted_links = dict((k, UrlEncoded(v, skip_encode=True)) - for k, v in results['links'].items()) - results['links'] = unquoted_links - return results - - def reload(self): - """Reloads the entity.""" - self.post("_reload") - return self - - def acl_update(self, **kwargs): - """To update Access Control List (ACL) properties for an endpoint. - - :param kwargs: Additional entity-specific arguments (required). - - - "owner" (``string``): The Splunk username, such as "admin". A value of "nobody" means no specific user (required). - - - "sharing" (``string``): A mode that indicates how the resource is shared. The sharing mode can be "user", "app", "global", or "system" (required). - - :type kwargs: ``dict`` - - **Example**:: - - import splunklib.client as client - service = client.connect(...) - saved_search = service.saved_searches["name"] - saved_search.acl_update(sharing="app", owner="nobody", app="search", **{"perms.read": "admin, nobody"}) - """ - if "body" not in kwargs: - kwargs = {"body": kwargs} - - if "sharing" not in kwargs["body"]: - raise ValueError("Required argument 'sharing' is missing.") - if "owner" not in kwargs["body"]: - raise ValueError("Required argument 'owner' is missing.") - - self.post("acl", **kwargs) - self.refresh() - return self - - @property - def state(self): - """Returns the entity's state record. - - :return: A ``dict`` containing fields and metadata for the entity. - """ - if self._state is None: self.refresh() - return self._state - - def update(self, **kwargs): - """Updates the server with any changes you've made to the current entity - along with any additional arguments you specify. - - **Note**: You cannot update the ``name`` field of an entity. - - Many of the fields in the REST API are not valid Python - identifiers, which means you cannot pass them as keyword - arguments. That is, Python will fail to parse the following:: - - # This fails - x.update(check-new=False, email.to='boris@utopia.net') - - However, you can always explicitly use a dictionary to pass - such keys:: - - # This works - x.update(**{'check-new': False, 'email.to': 'boris@utopia.net'}) - - :param kwargs: Additional entity-specific arguments (optional). - :type kwargs: ``dict`` - - :return: The entity this method is called on. - :rtype: class:`Entity` - """ - # The peculiarity in question: the REST API creates a new - # Entity if we pass name in the dictionary, instead of the - # expected behavior of updating this Entity. Therefore, we - # check for 'name' in kwargs and throw an error if it is - # there. - if 'name' in kwargs: - raise IllegalOperationException('Cannot update the name of an Entity via the REST API.') - self.post(**kwargs) - return self - - -class ReadOnlyCollection(Endpoint): - """This class represents a read-only collection of entities in the Splunk - instance. - """ - - def __init__(self, service, path, item=Entity): - Endpoint.__init__(self, service, path) - self.item = item # Item accessor - self.null_count = -1 - - def __contains__(self, name): - """Is there at least one entry called *name* in this collection? - - Makes a single roundtrip to the server, plus at most two more - if - the ``autologin`` field of :func:`connect` is set to ``True``. - """ - try: - self[name] - return True - except KeyError: - return False - except AmbiguousReferenceException: - return True - - def __getitem__(self, key): - """Fetch an item named *key* from this collection. - - A name is not a unique identifier in a collection. The unique - identifier is a name plus a namespace. For example, there can - be a saved search named ``'mysearch'`` with sharing ``'app'`` - in application ``'search'``, and another with sharing - ``'user'`` with owner ``'boris'`` and application - ``'search'``. If the ``Collection`` is attached to a - ``Service`` that has ``'-'`` (wildcard) as user and app in its - namespace, then both of these may be visible under the same - name. - - Where there is no conflict, ``__getitem__`` will fetch the - entity given just the name. If there is a conflict, and you - pass just a name, it will raise a ``ValueError``. In that - case, add the namespace as a second argument. - - This function makes a single roundtrip to the server, plus at - most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param key: The name to fetch, or a tuple (name, namespace). - :return: An :class:`Entity` object. - :raises KeyError: Raised if *key* does not exist. - :raises ValueError: Raised if no namespace is specified and *key* - does not refer to a unique name. - - **Example**:: - - s = client.connect(...) - saved_searches = s.saved_searches - x1 = saved_searches.create( - 'mysearch', 'search * | head 1', - owner='admin', app='search', sharing='app') - x2 = saved_searches.create( - 'mysearch', 'search * | head 1', - owner='admin', app='search', sharing='user') - # Raises ValueError: - saved_searches['mysearch'] - # Fetches x1 - saved_searches[ - 'mysearch', - client.namespace(sharing='app', app='search')] - # Fetches x2 - saved_searches[ - 'mysearch', - client.namespace(sharing='user', owner='boris', app='search')] - """ - try: - if isinstance(key, tuple) and len(key) == 2: - # x[a,b] is translated to x.__getitem__( (a,b) ), so we - # have to extract values out. - key, ns = key - key = UrlEncoded(key, encode_slash=True) - response = self.get(key, owner=ns.owner, app=ns.app) - else: - key = UrlEncoded(key, encode_slash=True) - response = self.get(key) - entries = self._load_list(response) - if len(entries) > 1: - raise AmbiguousReferenceException( - f"Found multiple entities named '{key}'; please specify a namespace.") - if len(entries) == 0: - raise KeyError(key) - return entries[0] - except HTTPError as he: - if he.status == 404: # No entity matching key and namespace. - raise KeyError(key) - else: - raise - - def __iter__(self, **kwargs): - """Iterate over the entities in the collection. - - :param kwargs: Additional arguments. - :type kwargs: ``dict`` - :rtype: iterator over entities. - - Implemented to give Collection a listish interface. This - function always makes a roundtrip to the server, plus at most - two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - **Example**:: - - import splunklib.client as client - c = client.connect(...) - saved_searches = c.saved_searches - for entity in saved_searches: - print(f"Saved search named {entity.name}") - """ - - for item in self.iter(**kwargs): - yield item - - def __len__(self): - """Enable ``len(...)`` for ``Collection`` objects. - - Implemented for consistency with a listish interface. No - further failure modes beyond those possible for any method on - an Endpoint. - - This function always makes a round trip to the server, plus at - most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - **Example**:: - - import splunklib.client as client - c = client.connect(...) - saved_searches = c.saved_searches - n = len(saved_searches) - """ - return len(self.list()) - - def _entity_path(self, state): - """Calculate the path to an entity to be returned. - - *state* should be the dictionary returned by - :func:`_parse_atom_entry`. :func:`_entity_path` extracts the - link to this entity from *state*, and strips all the namespace - prefixes from it to leave only the relative path of the entity - itself, sans namespace. - - :rtype: ``string`` - :return: an absolute path - """ - # This has been factored out so that it can be easily - # overloaded by Configurations, which has to switch its - # entities' endpoints from its own properties/ to configs/. - raw_path = parse.unquote(state.links.alternate) - if 'servicesNS/' in raw_path: - return _trailing(raw_path, 'servicesNS/', '/', '/') - if 'services/' in raw_path: - return _trailing(raw_path, 'services/') - return raw_path - - def _load_list(self, response): - """Converts *response* to a list of entities. - - *response* is assumed to be a :class:`Record` containing an - HTTP response, of the form:: - - {'status': 200, - 'headers': [('content-length', '232642'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Tue, 29 May 2012 15:27:08 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'OK', - 'body': ...a stream implementing .read()...} - - The ``'body'`` key refers to a stream containing an Atom feed, - that is, an XML document with a toplevel element ````, - and within that element one or more ```` elements. - """ - # Some subclasses of Collection have to override this because - # splunkd returns something that doesn't match - # . - entries = _load_atom_entries(response) - if entries is None: return [] - entities = [] - for entry in entries: - state = _parse_atom_entry(entry) - entity = self.item( - self.service, - self._entity_path(state), - state=state) - entities.append(entity) - - return entities - - def itemmeta(self): - """Returns metadata for members of the collection. - - Makes a single roundtrip to the server, plus two more at most if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :return: A :class:`splunklib.data.Record` object containing the metadata. - - **Example**:: - - import splunklib.client as client - import pprint - s = client.connect(...) - pprint.pprint(s.apps.itemmeta()) - {'access': {'app': 'search', - 'can_change_perms': '1', - 'can_list': '1', - 'can_share_app': '1', - 'can_share_global': '1', - 'can_share_user': '1', - 'can_write': '1', - 'modifiable': '1', - 'owner': 'admin', - 'perms': {'read': ['*'], 'write': ['admin']}, - 'removable': '0', - 'sharing': 'user'}, - 'fields': {'optional': ['author', - 'configured', - 'description', - 'label', - 'manageable', - 'template', - 'visible'], - 'required': ['name'], 'wildcard': []}} - """ - response = self.get("_new") - content = _load_atom(response, MATCH_ENTRY_CONTENT) - return _parse_atom_metadata(content) - - def iter(self, offset=0, count=None, pagesize=None, **kwargs): - """Iterates over the collection. - - This method is equivalent to the :meth:`list` method, but - it returns an iterator and can load a certain number of entities at a - time from the server. - - :param offset: The index of the first entity to return (optional). - :type offset: ``integer`` - :param count: The maximum number of entities to return (optional). - :type count: ``integer`` - :param pagesize: The number of entities to load (optional). - :type pagesize: ``integer`` - :param kwargs: Additional arguments (optional): - - - "search" (``string``): The search query to filter responses. - - - "sort_dir" (``string``): The direction to sort returned items: - "asc" or "desc". - - - "sort_key" (``string``): The field to use for sorting (optional). - - - "sort_mode" (``string``): The collating sequence for sorting - returned items: "auto", "alpha", "alpha_case", or "num". - - :type kwargs: ``dict`` - - **Example**:: - - import splunklib.client as client - s = client.connect(...) - for saved_search in s.saved_searches.iter(pagesize=10): - # Loads 10 saved searches at a time from the - # server. - ... - """ - assert pagesize is None or pagesize > 0 - if count is None: - count = self.null_count - fetched = 0 - while count == self.null_count or fetched < count: - response = self.get(count=pagesize or count, offset=offset, **kwargs) - items = self._load_list(response) - N = len(items) - fetched += N - for item in items: - yield item - if pagesize is None or N < pagesize: - break - offset += N - logger.debug("pagesize=%d, fetched=%d, offset=%d, N=%d, kwargs=%s", pagesize, fetched, offset, N, kwargs) - - # kwargs: count, offset, search, sort_dir, sort_key, sort_mode - def list(self, count=None, **kwargs): - """Retrieves a list of entities in this collection. - - The entire collection is loaded at once and is returned as a list. This - function makes a single roundtrip to the server, plus at most two more if - the ``autologin`` field of :func:`connect` is set to ``True``. - There is no caching--every call makes at least one round trip. - - :param count: The maximum number of entities to return (optional). - :type count: ``integer`` - :param kwargs: Additional arguments (optional): - - - "offset" (``integer``): The offset of the first item to return. - - - "search" (``string``): The search query to filter responses. - - - "sort_dir" (``string``): The direction to sort returned items: - "asc" or "desc". - - - "sort_key" (``string``): The field to use for sorting (optional). - - - "sort_mode" (``string``): The collating sequence for sorting - returned items: "auto", "alpha", "alpha_case", or "num". - - :type kwargs: ``dict`` - :return: A ``list`` of entities. - """ - # response = self.get(count=count, **kwargs) - # return self._load_list(response) - return list(self.iter(count=count, **kwargs)) - - -class Collection(ReadOnlyCollection): - """A collection of entities. - - Splunk provides a number of different collections of distinct - entity types: applications, saved searches, fired alerts, and a - number of others. Each particular type is available separately - from the Splunk instance, and the entities of that type are - returned in a :class:`Collection`. - - The interface for :class:`Collection` does not quite match either - ``list`` or ``dict`` in Python, because there are enough semantic - mismatches with either to make its behavior surprising. A unique - element in a :class:`Collection` is defined by a string giving its - name plus namespace (although the namespace is optional if the name is - unique). - - **Example**:: - - import splunklib.client as client - service = client.connect(...) - mycollection = service.saved_searches - mysearch = mycollection['my_search', client.namespace(owner='boris', app='natasha', sharing='user')] - # Or if there is only one search visible named 'my_search' - mysearch = mycollection['my_search'] - - Similarly, ``name`` in ``mycollection`` works as you might expect (though - you cannot currently pass a namespace to the ``in`` operator), as does - ``len(mycollection)``. - - However, as an aggregate, :class:`Collection` behaves more like a - list. If you iterate over a :class:`Collection`, you get an - iterator over the entities, not the names and namespaces. - - **Example**:: - - for entity in mycollection: - assert isinstance(entity, client.Entity) - - Use the :meth:`create` and :meth:`delete` methods to create and delete - entities in this collection. To view the access control list and other - metadata of the collection, use the :meth:`ReadOnlyCollection.itemmeta` method. - - :class:`Collection` does no caching. Each call makes at least one - round trip to the server to fetch data. - """ - - def create(self, name, **params): - """Creates a new entity in this collection. - - This function makes either one or two roundtrips to the - server, depending on the type of entities in this - collection, plus at most two more if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param name: The name of the entity to create. - :type name: ``string`` - :param namespace: A namespace, as created by the :func:`splunklib.binding.namespace` - function (optional). You can also set ``owner``, ``app``, and - ``sharing`` in ``params``. - :type namespace: A :class:`splunklib.data.Record` object with keys ``owner``, ``app``, - and ``sharing``. - :param params: Additional entity-specific arguments (optional). - :type params: ``dict`` - :return: The new entity. - :rtype: A subclass of :class:`Entity`, chosen by :meth:`Collection.self.item`. - - **Example**:: - - import splunklib.client as client - s = client.connect(...) - applications = s.apps - new_app = applications.create("my_fake_app") - """ - if not isinstance(name, str): - raise InvalidNameException(f"{name} is not a valid name for an entity.") - if 'namespace' in params: - namespace = params.pop('namespace') - params['owner'] = namespace.owner - params['app'] = namespace.app - params['sharing'] = namespace.sharing - response = self.post(name=name, **params) - atom = _load_atom(response, XNAME_ENTRY) - if atom is None: - # This endpoint doesn't return the content of the new - # item. We have to go fetch it ourselves. - return self[name] - entry = atom.entry - state = _parse_atom_entry(entry) - entity = self.item( - self.service, - self._entity_path(state), - state=state) - return entity - - def delete(self, name, **params): - """Deletes a specified entity from the collection. - - :param name: The name of the entity to delete. - :type name: ``string`` - :return: The collection. - :rtype: ``self`` - - This method is implemented for consistency with the REST API's DELETE - method. - - If there is no *name* entity on the server, a ``KeyError`` is - thrown. This function always makes a roundtrip to the server. - - **Example**:: - - import splunklib.client as client - c = client.connect(...) - saved_searches = c.saved_searches - saved_searches.create('my_saved_search', - 'search * | head 1') - assert 'my_saved_search' in saved_searches - saved_searches.delete('my_saved_search') - assert 'my_saved_search' not in saved_searches - """ - name = UrlEncoded(name, encode_slash=True) - if 'namespace' in params: - namespace = params.pop('namespace') - params['owner'] = namespace.owner - params['app'] = namespace.app - params['sharing'] = namespace.sharing - try: - self.service.delete(_path(self.path, name), **params) - except HTTPError as he: - # An HTTPError with status code 404 means that the entity - # has already been deleted, and we reraise it as a - # KeyError. - if he.status == 404: - raise KeyError(f"No such entity {name}") - else: - raise - return self - - def get(self, name="", owner=None, app=None, sharing=None, **query): - """Performs a GET request to the server on the collection. - - If *owner*, *app*, and *sharing* are omitted, this method takes a - default namespace from the :class:`Service` object for this :class:`Endpoint`. - All other keyword arguments are included in the URL as query parameters. - - :raises AuthenticationError: Raised when the ``Service`` is not logged in. - :raises HTTPError: Raised when an error in the request occurs. - :param path_segment: A path segment relative to this endpoint. - :type path_segment: ``string`` - :param owner: The owner context of the namespace (optional). - :type owner: ``string`` - :param app: The app context of the namespace (optional). - :type app: ``string`` - :param sharing: The sharing mode for the namespace (optional). - :type sharing: "global", "system", "app", or "user" - :param query: All other keyword arguments, which are used as query - parameters. - :type query: ``string`` - :return: The response from the server. - :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, - and ``status`` - - **Example**:: - - import splunklib.client - s = client.service(...) - saved_searches = s.saved_searches - saved_searches.get("my/saved/search") == \\ - {'body': ...a response reader object..., - 'headers': [('content-length', '26208'), - ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), - ('server', 'Splunkd'), - ('connection', 'close'), - ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), - ('date', 'Fri, 11 May 2012 16:30:35 GMT'), - ('content-type', 'text/xml; charset=utf-8')], - 'reason': 'OK', - 'status': 200} - saved_searches.get('nonexistant/search') # raises HTTPError - s.logout() - saved_searches.get() # raises AuthenticationError - - """ - name = UrlEncoded(name, encode_slash=True) - return super().get(name, owner, app, sharing, **query) - - -class ConfigurationFile(Collection): - """This class contains all of the stanzas from one configuration file. - """ - - # __init__'s arguments must match those of an Entity, not a - # Collection, since it is being created as the elements of a - # Configurations, which is a Collection subclass. - def __init__(self, service, path, **kwargs): - Collection.__init__(self, service, path, item=Stanza) - self.name = kwargs['state']['title'] - - -class Configurations(Collection): - """This class provides access to the configuration files from this Splunk - instance. Retrieve this collection using :meth:`Service.confs`. - - Splunk's configuration is divided into files, and each file into - stanzas. This collection is unusual in that the values in it are - themselves collections of :class:`ConfigurationFile` objects. - """ - - def __init__(self, service): - Collection.__init__(self, service, PATH_PROPERTIES, item=ConfigurationFile) - if self.service.namespace.owner == '-' or self.service.namespace.app == '-': - raise ValueError("Configurations cannot have wildcards in namespace.") - - def __getitem__(self, key): - # The superclass implementation is designed for collections that contain - # entities. This collection (Configurations) contains collections - # (ConfigurationFile). - # - # The configurations endpoint returns multiple entities when we ask for a single file. - # This screws up the default implementation of __getitem__ from Collection, which thinks - # that multiple entities means a name collision, so we have to override it here. - try: - self.get(key) - return ConfigurationFile(self.service, PATH_CONF % key, state={'title': key}) - except HTTPError as he: - if he.status == 404: # No entity matching key - raise KeyError(key) - else: - raise - - def __contains__(self, key): - # configs/conf-{name} never returns a 404. We have to post to properties/{name} - # in order to find out if a configuration exists. - try: - self.get(key) - return True - except HTTPError as he: - if he.status == 404: # No entity matching key - return False - raise - - def create(self, name): - """ Creates a configuration file named *name*. - - If there is already a configuration file with that name, - the existing file is returned. - - :param name: The name of the configuration file. - :type name: ``string`` - - :return: The :class:`ConfigurationFile` object. - """ - # This has to be overridden to handle the plumbing of creating - # a ConfigurationFile (which is a Collection) instead of some - # Entity. - if not isinstance(name, str): - raise ValueError(f"Invalid name: {repr(name)}") - response = self.post(__conf=name) - if response.status == 303: - return self[name] - if response.status == 201: - return ConfigurationFile(self.service, PATH_CONF % name, item=Stanza, state={'title': name}) - raise ValueError(f"Unexpected status code {response.status} returned from creating a stanza") - - def delete(self, key): - """Raises `IllegalOperationException`.""" - raise IllegalOperationException("Cannot delete configuration files from the REST API.") - - def _entity_path(self, state): - # Overridden to make all the ConfigurationFile objects - # returned refer to the configs/ path instead of the - # properties/ path used by Configrations. - return PATH_CONF % state['title'] - - -class Stanza(Entity): - """This class contains a single configuration stanza.""" - - def submit(self, stanza): - """Adds keys to the current configuration stanza as a - dictionary of key-value pairs. - - :param stanza: A dictionary of key-value pairs for the stanza. - :type stanza: ``dict`` - :return: The :class:`Stanza` object. - """ - body = _encode(**stanza) - self.service.post(self.path, body=body) - return self - - def __len__(self): - # The stanza endpoint returns all the keys at the same level in the XML as the eai information - # and 'disabled', so to get an accurate length, we have to filter those out and have just - # the stanza keys. - return len([x for x in self._state.content.keys() - if not x.startswith('eai') and x != 'disabled']) - - -class StoragePassword(Entity): - """This class contains a storage password. - """ - - def __init__(self, service, path, **kwargs): - state = kwargs.get('state', None) - kwargs['skip_refresh'] = kwargs.get('skip_refresh', state is not None) - super().__init__(service, path, **kwargs) - self._state = state - - @property - def clear_password(self): - return self.content.get('clear_password') - - @property - def encrypted_password(self): - return self.content.get('encr_password') - - @property - def realm(self): - return self.content.get('realm') - - @property - def username(self): - return self.content.get('username') - - -class StoragePasswords(Collection): - """This class provides access to the storage passwords from this Splunk - instance. Retrieve this collection using :meth:`Service.storage_passwords`. - """ - - def __init__(self, service): - if service.namespace.owner == '-' or service.namespace.app == '-': - raise ValueError("StoragePasswords cannot have wildcards in namespace.") - super().__init__(service, PATH_STORAGE_PASSWORDS, item=StoragePassword) - - def create(self, password, username, realm=None): - """ Creates a storage password. - - A `StoragePassword` can be identified by , or by : if the - optional realm parameter is also provided. - - :param password: The password for the credentials - this is the only part of the credentials that will be stored securely. - :type name: ``string`` - :param username: The username for the credentials. - :type name: ``string`` - :param realm: The credential realm. (optional) - :type name: ``string`` - - :return: The :class:`StoragePassword` object created. - """ - if not isinstance(username, str): - raise ValueError(f"Invalid name: {repr(username)}") - - if realm is None: - response = self.post(password=password, name=username) - else: - response = self.post(password=password, realm=realm, name=username) - - if response.status != 201: - raise ValueError(f"Unexpected status code {response.status} returned from creating a stanza") - - entries = _load_atom_entries(response) - state = _parse_atom_entry(entries[0]) - storage_password = StoragePassword(self.service, self._entity_path(state), state=state, skip_refresh=True) - - return storage_password - - def delete(self, username, realm=None): - """Delete a storage password by username and/or realm. - - The identifier can be passed in through the username parameter as - or :, but the preferred way is by - passing in the username and realm parameters. - - :param username: The username for the credentials, or : if the realm parameter is omitted. - :type name: ``string`` - :param realm: The credential realm. (optional) - :type name: ``string`` - :return: The `StoragePassword` collection. - :rtype: ``self`` - """ - if realm is None: - # This case makes the username optional, so - # the full name can be passed in as realm. - # Assume it's already encoded. - name = username - else: - # Encode each component separately - name = UrlEncoded(realm, encode_slash=True) + ":" + UrlEncoded(username, encode_slash=True) - - # Append the : expected at the end of the name - if name[-1] != ":": - name = name + ":" - return Collection.delete(self, name) - - -class AlertGroup(Entity): - """This class represents a group of fired alerts for a saved search. Access - it using the :meth:`alerts` property.""" - - def __init__(self, service, path, **kwargs): - Entity.__init__(self, service, path, **kwargs) - - def __len__(self): - return self.count - - @property - def alerts(self): - """Returns a collection of triggered alerts. - - :return: A :class:`Collection` of triggered alerts. - """ - return Collection(self.service, self.path) - - @property - def count(self): - """Returns the count of triggered alerts. - - :return: The triggered alert count. - :rtype: ``integer`` - """ - return int(self.content.get('triggered_alert_count', 0)) - - -class Indexes(Collection): - """This class contains the collection of indexes in this Splunk instance. - Retrieve this collection using :meth:`Service.indexes`. - """ - - def get_default(self): - """ Returns the name of the default index. - - :return: The name of the default index. - - """ - index = self['_audit'] - return index['defaultDatabase'] - - def delete(self, name): - """ Deletes a given index. - - **Note**: This method is only supported in Splunk 5.0 and later. - - :param name: The name of the index to delete. - :type name: ``string`` - """ - if self.service.splunk_version >= (5,): - Collection.delete(self, name) - else: - raise IllegalOperationException("Deleting indexes via the REST API is " - "not supported before Splunk version 5.") - - -class Index(Entity): - """This class represents an index and provides different operations, such as - cleaning the index, writing to the index, and so forth.""" - - def __init__(self, service, path, **kwargs): - Entity.__init__(self, service, path, **kwargs) - - def attach(self, host=None, source=None, sourcetype=None): - """Opens a stream (a writable socket) for writing events to the index. - - :param host: The host value for events written to the stream. - :type host: ``string`` - :param source: The source value for events written to the stream. - :type source: ``string`` - :param sourcetype: The sourcetype value for events written to the - stream. - :type sourcetype: ``string`` - - :return: A writable socket. - """ - args = {'index': self.name} - if host is not None: args['host'] = host - if source is not None: args['source'] = source - if sourcetype is not None: args['sourcetype'] = sourcetype - path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + parse.urlencode(args), skip_encode=True) - - cookie_header = self.service.token if self.service.token is _NoAuthenticationToken else self.service.token.replace("Splunk ", "") - cookie_or_auth_header = f"Authorization: Splunk {cookie_header}\r\n" - - # If we have cookie(s), use them instead of "Authorization: ..." - if self.service.has_cookies(): - cookie_header = _make_cookie_header(self.service.get_cookies().items()) - cookie_or_auth_header = f"Cookie: {cookie_header}\r\n" - - # Since we need to stream to the index connection, we have to keep - # the connection open and use the Splunk extension headers to note - # the input mode - sock = self.service.connect() - headers = [f"POST {str(self.service._abspath(path))} HTTP/1.1\r\n".encode('utf-8'), - f"Host: {self.service.host}:{int(self.service.port)}\r\n".encode('utf-8'), - b"Accept-Encoding: identity\r\n", - cookie_or_auth_header.encode('utf-8'), - b"X-Splunk-Input-Mode: Streaming\r\n", - b"\r\n"] - - for h in headers: - sock.write(h) - return sock - - @contextlib.contextmanager - def attached_socket(self, *args, **kwargs): - """Opens a raw socket in a ``with`` block to write data to Splunk. - - The arguments are identical to those for :meth:`attach`. The socket is - automatically closed at the end of the ``with`` block, even if an - exception is raised in the block. - - :param host: The host value for events written to the stream. - :type host: ``string`` - :param source: The source value for events written to the stream. - :type source: ``string`` - :param sourcetype: The sourcetype value for events written to the - stream. - :type sourcetype: ``string`` - - :returns: Nothing. - - **Example**:: - - import splunklib.client as client - s = client.connect(...) - index = s.indexes['some_index'] - with index.attached_socket(sourcetype='test') as sock: - sock.send('Test event\\r\\n') - - """ - try: - sock = self.attach(*args, **kwargs) - yield sock - finally: - sock.shutdown(socket.SHUT_RDWR) - sock.close() - - def clean(self, timeout=60): - """Deletes the contents of the index. - - This method blocks until the index is empty, because it needs to restore - values at the end of the operation. - - :param timeout: The time-out period for the operation, in seconds (the - default is 60). - :type timeout: ``integer`` - - :return: The :class:`Index`. - """ - self.refresh() - - tds = self['maxTotalDataSizeMB'] - ftp = self['frozenTimePeriodInSecs'] - was_disabled_initially = self.disabled - try: - if not was_disabled_initially and self.service.splunk_version < (5,): - # Need to disable the index first on Splunk 4.x, - # but it doesn't work to disable it on 5.0. - self.disable() - self.update(maxTotalDataSizeMB=1, frozenTimePeriodInSecs=1) - self.roll_hot_buckets() - - # Wait until event count goes to 0. - start = datetime.now() - diff = timedelta(seconds=timeout) - while self.content.totalEventCount != '0' and datetime.now() < start + diff: - sleep(1) - self.refresh() - - if self.content.totalEventCount != '0': - raise OperationError( - f"Cleaning index {self.name} took longer than {timeout} seconds; timing out.") - finally: - # Restore original values - self.update(maxTotalDataSizeMB=tds, frozenTimePeriodInSecs=ftp) - if not was_disabled_initially and self.service.splunk_version < (5,): - # Re-enable the index if it was originally enabled and we messed with it. - self.enable() - - return self - - def roll_hot_buckets(self): - """Performs rolling hot buckets for this index. - - :return: The :class:`Index`. - """ - self.post("roll-hot-buckets") - return self - - def submit(self, event, host=None, source=None, sourcetype=None): - """Submits a single event to the index using ``HTTP POST``. - - :param event: The event to submit. - :type event: ``string`` - :param `host`: The host value of the event. - :type host: ``string`` - :param `source`: The source value of the event. - :type source: ``string`` - :param `sourcetype`: The sourcetype value of the event. - :type sourcetype: ``string`` - - :return: The :class:`Index`. - """ - args = {'index': self.name} - if host is not None: args['host'] = host - if source is not None: args['source'] = source - if sourcetype is not None: args['sourcetype'] = sourcetype - - self.service.post(PATH_RECEIVERS_SIMPLE, body=event, **args) - return self - - # kwargs: host, host_regex, host_segment, rename-source, sourcetype - def upload(self, filename, **kwargs): - """Uploads a file for immediate indexing. - - **Note**: The file must be locally accessible from the server. - - :param filename: The name of the file to upload. The file can be a - plain, compressed, or archived file. - :type filename: ``string`` - :param kwargs: Additional arguments (optional). For more about the - available parameters, see `Index parameters `_ on Splunk Developer Portal. - :type kwargs: ``dict`` - - :return: The :class:`Index`. - """ - kwargs['index'] = self.name - path = 'data/inputs/oneshot' - self.service.post(path, name=filename, **kwargs) - return self - - -class Input(Entity): - """This class represents a Splunk input. This class is the base for all - typed input classes and is also used when the client does not recognize an - input kind. - """ - - def __init__(self, service, path, kind=None, **kwargs): - # kind can be omitted (in which case it is inferred from the path) - # Otherwise, valid values are the paths from data/inputs ("udp", - # "monitor", "tcp/raw"), or two special cases: "tcp" (which is "tcp/raw") - # and "splunktcp" (which is "tcp/cooked"). - Entity.__init__(self, service, path, **kwargs) - if kind is None: - path_segments = path.split('/') - i = path_segments.index('inputs') + 1 - if path_segments[i] == 'tcp': - self.kind = path_segments[i] + '/' + path_segments[i + 1] - else: - self.kind = path_segments[i] - else: - self.kind = kind - - # Handle old input kind names. - if self.kind == 'tcp': - self.kind = 'tcp/raw' - if self.kind == 'splunktcp': - self.kind = 'tcp/cooked' - - def update(self, **kwargs): - """Updates the server with any changes you've made to the current input - along with any additional arguments you specify. - - :param kwargs: Additional arguments (optional). For more about the - available parameters, see `Input parameters `_ on Splunk Developer Portal. - :type kwargs: ``dict`` - - :return: The input this method was called on. - :rtype: class:`Input` - """ - # UDP and TCP inputs require special handling due to their restrictToHost - # field. For all other inputs kinds, we can dispatch to the superclass method. - if self.kind not in ['tcp', 'splunktcp', 'tcp/raw', 'tcp/cooked', 'udp']: - return super().update(**kwargs) - else: - # The behavior of restrictToHost is inconsistent across input kinds and versions of Splunk. - # In Splunk 4.x, the name of the entity is only the port, independent of the value of - # restrictToHost. In Splunk 5.0 this changed so the name will be of the form :. - # In 5.0 and 5.0.1, if you don't supply the restrictToHost value on every update, it will - # remove the host restriction from the input. As of 5.0.2 you simply can't change restrictToHost - # on an existing input. - - # The logic to handle all these cases: - # - Throw an exception if the user tries to set restrictToHost on an existing input - # for *any* version of Splunk. - # - Set the existing restrictToHost value on the update args internally so we don't - # cause it to change in Splunk 5.0 and 5.0.1. - to_update = kwargs.copy() - - if 'restrictToHost' in kwargs: - raise IllegalOperationException("Cannot set restrictToHost on an existing input with the SDK.") - if 'restrictToHost' in self._state.content and self.kind != 'udp': - to_update['restrictToHost'] = self._state.content['restrictToHost'] - - # Do the actual update operation. - return super().update(**to_update) - - -# Inputs is a "kinded" collection, which is a heterogenous collection where -# each item is tagged with a kind, that provides a single merged view of all -# input kinds. -class Inputs(Collection): - """This class represents a collection of inputs. The collection is - heterogeneous and each member of the collection contains a *kind* property - that indicates the specific type of input. - Retrieve this collection using :meth:`Service.inputs`.""" - - def __init__(self, service, kindmap=None): - Collection.__init__(self, service, PATH_INPUTS, item=Input) - - def __getitem__(self, key): - # The key needed to retrieve the input needs it's parenthesis to be URL encoded - # based on the REST API for input - # - if isinstance(key, tuple) and len(key) == 2: - # Fetch a single kind - key, kind = key - key = UrlEncoded(key, encode_slash=True) - try: - response = self.get(self.kindpath(kind) + "/" + key) - entries = self._load_list(response) - if len(entries) > 1: - raise AmbiguousReferenceException(f"Found multiple inputs of kind {kind} named {key}.") - if len(entries) == 0: - raise KeyError((key, kind)) - return entries[0] - except HTTPError as he: - if he.status == 404: # No entity matching kind and key - raise KeyError((key, kind)) - else: - raise - else: - # Iterate over all the kinds looking for matches. - kind = None - candidate = None - key = UrlEncoded(key, encode_slash=True) - for kind in self.kinds: - try: - response = self.get(kind + "/" + key) - entries = self._load_list(response) - if len(entries) > 1: - raise AmbiguousReferenceException(f"Found multiple inputs of kind {kind} named {key}.") - if len(entries) == 0: - pass - else: - if candidate is not None: # Already found at least one candidate - raise AmbiguousReferenceException( - f"Found multiple inputs named {key}, please specify a kind") - candidate = entries[0] - except HTTPError as he: - if he.status == 404: - pass # Just carry on to the next kind. - else: - raise - if candidate is None: - raise KeyError(key) # Never found a match. - return candidate - - def __contains__(self, key): - if isinstance(key, tuple) and len(key) == 2: - # If we specify a kind, this will shortcut properly - try: - self.__getitem__(key) - return True - except KeyError: - return False - else: - # Without a kind, we want to minimize the number of round trips to the server, so we - # reimplement some of the behavior of __getitem__ in order to be able to stop searching - # on the first hit. - for kind in self.kinds: - try: - response = self.get(self.kindpath(kind) + "/" + key) - entries = self._load_list(response) - if len(entries) > 0: - return True - except HTTPError as he: - if he.status == 404: - pass # Just carry on to the next kind. - else: - raise - return False - - def create(self, name, kind, **kwargs): - """Creates an input of a specific kind in this collection, with any - arguments you specify. - - :param `name`: The input name. - :type name: ``string`` - :param `kind`: The kind of input: - - - "ad": Active Directory - - - "monitor": Files and directories - - - "registry": Windows Registry - - - "script": Scripts - - - "splunktcp": TCP, processed - - - "tcp": TCP, unprocessed - - - "udp": UDP - - - "win-event-log-collections": Windows event log - - - "win-perfmon": Performance monitoring - - - "win-wmi-collections": WMI - - :type kind: ``string`` - :param `kwargs`: Additional arguments (optional). For more about the - available parameters, see `Input parameters `_ on Splunk Developer Portal. - - :type kwargs: ``dict`` - - :return: The new :class:`Input`. - """ - kindpath = self.kindpath(kind) - self.post(kindpath, name=name, **kwargs) - - # If we created an input with restrictToHost set, then - # its path will be :, not just , - # and we have to adjust accordingly. - - # Url encodes the name of the entity. - name = UrlEncoded(name, encode_slash=True) - path = _path( - self.path + kindpath, - f"{kwargs['restrictToHost']}:{name}" if 'restrictToHost' in kwargs else name - ) - return Input(self.service, path, kind) - - def delete(self, name, kind=None): - """Removes an input from the collection. - - :param `kind`: The kind of input: - - - "ad": Active Directory - - - "monitor": Files and directories - - - "registry": Windows Registry - - - "script": Scripts - - - "splunktcp": TCP, processed - - - "tcp": TCP, unprocessed - - - "udp": UDP - - - "win-event-log-collections": Windows event log - - - "win-perfmon": Performance monitoring - - - "win-wmi-collections": WMI - - :type kind: ``string`` - :param name: The name of the input to remove. - :type name: ``string`` - - :return: The :class:`Inputs` collection. - """ - if kind is None: - self.service.delete(self[name].path) - else: - self.service.delete(self[name, kind].path) - return self - - def itemmeta(self, kind): - """Returns metadata for the members of a given kind. - - :param `kind`: The kind of input: - - - "ad": Active Directory - - - "monitor": Files and directories - - - "registry": Windows Registry - - - "script": Scripts - - - "splunktcp": TCP, processed - - - "tcp": TCP, unprocessed - - - "udp": UDP - - - "win-event-log-collections": Windows event log - - - "win-perfmon": Performance monitoring - - - "win-wmi-collections": WMI - - :type kind: ``string`` - - :return: The metadata. - :rtype: class:``splunklib.data.Record`` - """ - response = self.get(f"{self._kindmap[kind]}/_new") - content = _load_atom(response, MATCH_ENTRY_CONTENT) - return _parse_atom_metadata(content) - - def _get_kind_list(self, subpath=None): - if subpath is None: - subpath = [] - - kinds = [] - response = self.get('/'.join(subpath)) - content = _load_atom_entries(response) - for entry in content: - this_subpath = subpath + [entry.title] - # The "all" endpoint doesn't work yet. - # The "tcp/ssl" endpoint is not a real input collection. - if entry.title == 'all' or this_subpath == ['tcp', 'ssl']: - continue - if 'create' in [x.rel for x in entry.link]: - path = '/'.join(subpath + [entry.title]) - kinds.append(path) - else: - subkinds = self._get_kind_list(subpath + [entry.title]) - kinds.extend(subkinds) - return kinds - - @property - def kinds(self): - """Returns the input kinds on this Splunk instance. - - :return: The list of input kinds. - :rtype: ``list`` - """ - return self._get_kind_list() - - def kindpath(self, kind): - """Returns a path to the resources for a given input kind. - - :param `kind`: The kind of input: - - - "ad": Active Directory - - - "monitor": Files and directories - - - "registry": Windows Registry - - - "script": Scripts - - - "splunktcp": TCP, processed - - - "tcp": TCP, unprocessed - - - "udp": UDP - - - "win-event-log-collections": Windows event log - - - "win-perfmon": Performance monitoring - - - "win-wmi-collections": WMI - - :type kind: ``string`` - - :return: The relative endpoint path. - :rtype: ``string`` - """ - if kind == 'tcp': - return UrlEncoded('tcp/raw', skip_encode=True) - if kind == 'splunktcp': - return UrlEncoded('tcp/cooked', skip_encode=True) - return UrlEncoded(kind, skip_encode=True) - - def list(self, *kinds, **kwargs): - """Returns a list of inputs that are in the :class:`Inputs` collection. - You can also filter by one or more input kinds. - - This function iterates over all possible inputs, regardless of any arguments you - specify. Because the :class:`Inputs` collection is the union of all the inputs of each - kind, this method implements parameters such as "count", "search", and so - on at the Python level once all the data has been fetched. The exception - is when you specify a single input kind, and then this method makes a single request - with the usual semantics for parameters. - - :param kinds: The input kinds to return (optional). - - - "ad": Active Directory - - - "monitor": Files and directories - - - "registry": Windows Registry - - - "script": Scripts - - - "splunktcp": TCP, processed - - - "tcp": TCP, unprocessed - - - "udp": UDP - - - "win-event-log-collections": Windows event log - - - "win-perfmon": Performance monitoring - - - "win-wmi-collections": WMI - - :type kinds: ``string`` - :param kwargs: Additional arguments (optional): - - - "count" (``integer``): The maximum number of items to return. - - - "offset" (``integer``): The offset of the first item to return. - - - "search" (``string``): The search query to filter responses. - - - "sort_dir" (``string``): The direction to sort returned items: - "asc" or "desc". - - - "sort_key" (``string``): The field to use for sorting (optional). - - - "sort_mode" (``string``): The collating sequence for sorting - returned items: "auto", "alpha", "alpha_case", or "num". - - :type kwargs: ``dict`` - - :return: A list of input kinds. - :rtype: ``list`` - """ - if len(kinds) == 0: - kinds = self.kinds - if len(kinds) == 1: - kind = kinds[0] - logger.debug("Inputs.list taking short circuit branch for single kind.") - path = self.kindpath(kind) - logger.debug("Path for inputs: %s", path) - try: - path = UrlEncoded(path, skip_encode=True) - response = self.get(path, **kwargs) - except HTTPError as he: - if he.status == 404: # No inputs of this kind - return [] - entities = [] - entries = _load_atom_entries(response) - if entries is None: - return [] # No inputs in a collection comes back with no feed or entry in the XML - for entry in entries: - state = _parse_atom_entry(entry) - # Unquote the URL, since all URL encoded in the SDK - # should be of type UrlEncoded, and all str should not - # be URL encoded. - path = parse.unquote(state.links.alternate) - entity = Input(self.service, path, kind, state=state) - entities.append(entity) - return entities - - search = kwargs.get('search', '*') - - entities = [] - for kind in kinds: - response = None - try: - kind = UrlEncoded(kind, skip_encode=True) - response = self.get(self.kindpath(kind), search=search) - except HTTPError as e: - if e.status == 404: - continue # No inputs of this kind - else: - raise - - entries = _load_atom_entries(response) - if entries is None: continue # No inputs to process - for entry in entries: - state = _parse_atom_entry(entry) - # Unquote the URL, since all URL encoded in the SDK - # should be of type UrlEncoded, and all str should not - # be URL encoded. - path = parse.unquote(state.links.alternate) - entity = Input(self.service, path, kind, state=state) - entities.append(entity) - if 'offset' in kwargs: - entities = entities[kwargs['offset']:] - if 'count' in kwargs: - entities = entities[:kwargs['count']] - if kwargs.get('sort_mode', None) == 'alpha': - sort_field = kwargs.get('sort_field', 'name') - if sort_field == 'name': - f = lambda x: x.name.lower() - else: - f = lambda x: x[sort_field].lower() - entities = sorted(entities, key=f) - if kwargs.get('sort_mode', None) == 'alpha_case': - sort_field = kwargs.get('sort_field', 'name') - if sort_field == 'name': - f = lambda x: x.name - else: - f = lambda x: x[sort_field] - entities = sorted(entities, key=f) - if kwargs.get('sort_dir', 'asc') == 'desc': - entities = list(reversed(entities)) - return entities - - def __iter__(self, **kwargs): - for item in self.iter(**kwargs): - yield item - - def iter(self, **kwargs): - """ Iterates over the collection of inputs. - - :param kwargs: Additional arguments (optional): - - - "count" (``integer``): The maximum number of items to return. - - - "offset" (``integer``): The offset of the first item to return. - - - "search" (``string``): The search query to filter responses. - - - "sort_dir" (``string``): The direction to sort returned items: - "asc" or "desc". - - - "sort_key" (``string``): The field to use for sorting (optional). - - - "sort_mode" (``string``): The collating sequence for sorting - returned items: "auto", "alpha", "alpha_case", or "num". - - :type kwargs: ``dict`` - """ - for item in self.list(**kwargs): - yield item - - def oneshot(self, path, **kwargs): - """ Creates a oneshot data input, which is an upload of a single file - for one-time indexing. - - :param path: The path and filename. - :type path: ``string`` - :param kwargs: Additional arguments (optional). For more about the - available parameters, see `Input parameters `_ on Splunk Developer Portal. - :type kwargs: ``dict`` - """ - self.post('oneshot', name=path, **kwargs) - - -class Job(Entity): - """This class represents a search job.""" - - def __init__(self, service, sid, **kwargs): - # Default to v2 in Splunk Version 9+ - path = "{path}{sid}" - # Formatting path based on the Splunk Version - if service.disable_v2_api: - path = path.format(path=PATH_JOBS, sid=sid) - else: - path = path.format(path=PATH_JOBS_V2, sid=sid) - - Entity.__init__(self, service, path, skip_refresh=True, **kwargs) - self.sid = sid - - # The Job entry record is returned at the root of the response - def _load_atom_entry(self, response): - return _load_atom(response).entry - - def cancel(self): - """Stops the current search and deletes the results cache. - - :return: The :class:`Job`. - """ - try: - self.post("control", action="cancel") - except HTTPError as he: - if he.status == 404: - # The job has already been cancelled, so - # cancelling it twice is a nop. - pass - else: - raise - return self - - def disable_preview(self): - """Disables preview for this job. - - :return: The :class:`Job`. - """ - self.post("control", action="disablepreview") - return self - - def enable_preview(self): - """Enables preview for this job. - - **Note**: Enabling preview might slow search considerably. - - :return: The :class:`Job`. - """ - self.post("control", action="enablepreview") - return self - - def events(self, **kwargs): - """Returns a streaming handle to this job's events. - - :param kwargs: Additional parameters (optional). For a list of valid - parameters, see `GET search/jobs/{search_id}/events - `_ - in the REST API documentation. - :type kwargs: ``dict`` - - :return: The ``InputStream`` IO handle to this job's events. - """ - kwargs['segmentation'] = kwargs.get('segmentation', 'none') - - # Search API v1(GET) and v2(POST) - if self.service.disable_v2_api: - return self.get("events", **kwargs).body - return self.post("events", **kwargs).body - - def finalize(self): - """Stops the job and provides intermediate results for retrieval. - - :return: The :class:`Job`. - """ - self.post("control", action="finalize") - return self - - def is_done(self): - """Indicates whether this job finished running. - - :return: ``True`` if the job is done, ``False`` if not. - :rtype: ``boolean`` - """ - if not self.is_ready(): - return False - done = (self._state.content['isDone'] == '1') - return done - - def is_ready(self): - """Indicates whether this job is ready for querying. - - :return: ``True`` if the job is ready, ``False`` if not. - :rtype: ``boolean`` - - """ - response = self.get() - if response.status == 204: - return False - self._state = self.read(response) - ready = self._state.content['dispatchState'] not in ['QUEUED', 'PARSING'] - return ready - - @property - def name(self): - """Returns the name of the search job, which is the search ID (SID). - - :return: The search ID. - :rtype: ``string`` - """ - return self.sid - - def pause(self): - """Suspends the current search. - - :return: The :class:`Job`. - """ - self.post("control", action="pause") - return self - - def results(self, **query_params): - """Returns a streaming handle to this job's search results. To get a nice, Pythonic iterator, pass the handle - to :class:`splunklib.results.JSONResultsReader` along with the query param "output_mode='json'", as in:: - - import splunklib.client as client - import splunklib.results as results - from time import sleep - service = client.connect(...) - job = service.jobs.create("search * | head 5") - while not job.is_done(): - sleep(.2) - rr = results.JSONResultsReader(job.results(output_mode='json')) - for result in rr: - if isinstance(result, results.Message): - # Diagnostic messages may be returned in the results - print(f'{result.type}: {result.message}') - elif isinstance(result, dict): - # Normal events are returned as dicts - print(result) - assert rr.is_preview == False - - Results are not available until the job has finished. If called on - an unfinished job, the result is an empty event set. - - This method makes a single roundtrip - to the server, plus at most two additional round trips if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param query_params: Additional parameters (optional). For a list of valid - parameters, see `GET search/jobs/{search_id}/results - `_. - :type query_params: ``dict`` - - :return: The ``InputStream`` IO handle to this job's results. - """ - query_params['segmentation'] = query_params.get('segmentation', 'none') - - # Search API v1(GET) and v2(POST) - if self.service.disable_v2_api: - return self.get("results", **query_params).body - return self.post("results", **query_params).body - - def preview(self, **query_params): - """Returns a streaming handle to this job's preview search results. - - Unlike :class:`splunklib.results.JSONResultsReader`along with the query param "output_mode='json'", - which requires a job to be finished to return any results, the ``preview`` method returns any results that - have been generated so far, whether the job is running or not. The returned search results are the raw data - from the server. Pass the handle returned to :class:`splunklib.results.JSONResultsReader` to get a nice, - Pythonic iterator over objects, as in:: - - import splunklib.client as client - import splunklib.results as results - service = client.connect(...) - job = service.jobs.create("search * | head 5") - rr = results.JSONResultsReader(job.preview(output_mode='json')) - for result in rr: - if isinstance(result, results.Message): - # Diagnostic messages may be returned in the results - print(f'{result.type}: {result.message}') - elif isinstance(result, dict): - # Normal events are returned as dicts - print(result) - if rr.is_preview: - print("Preview of a running search job.") - else: - print("Job is finished. Results are final.") - - This method makes one roundtrip to the server, plus at most - two more if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param query_params: Additional parameters (optional). For a list of valid - parameters, see `GET search/jobs/{search_id}/results_preview - `_ - in the REST API documentation. - :type query_params: ``dict`` - - :return: The ``InputStream`` IO handle to this job's preview results. - """ - query_params['segmentation'] = query_params.get('segmentation', 'none') - - # Search API v1(GET) and v2(POST) - if self.service.disable_v2_api: - return self.get("results_preview", **query_params).body - return self.post("results_preview", **query_params).body - - def searchlog(self, **kwargs): - """Returns a streaming handle to this job's search log. - - :param `kwargs`: Additional parameters (optional). For a list of valid - parameters, see `GET search/jobs/{search_id}/search.log - `_ - in the REST API documentation. - :type kwargs: ``dict`` - - :return: The ``InputStream`` IO handle to this job's search log. - """ - return self.get("search.log", **kwargs).body - - def set_priority(self, value): - """Sets this job's search priority in the range of 0-10. - - Higher numbers indicate higher priority. Unless splunkd is - running as *root*, you can only decrease the priority of a running job. - - :param `value`: The search priority. - :type value: ``integer`` - - :return: The :class:`Job`. - """ - self.post('control', action="setpriority", priority=value) - return self - - def summary(self, **kwargs): - """Returns a streaming handle to this job's summary. - - :param `kwargs`: Additional parameters (optional). For a list of valid - parameters, see `GET search/jobs/{search_id}/summary - `_ - in the REST API documentation. - :type kwargs: ``dict`` - - :return: The ``InputStream`` IO handle to this job's summary. - """ - return self.get("summary", **kwargs).body - - def timeline(self, **kwargs): - """Returns a streaming handle to this job's timeline results. - - :param `kwargs`: Additional timeline arguments (optional). For a list of valid - parameters, see `GET search/jobs/{search_id}/timeline - `_ - in the REST API documentation. - :type kwargs: ``dict`` - - :return: The ``InputStream`` IO handle to this job's timeline. - """ - return self.get("timeline", **kwargs).body - - def touch(self): - """Extends the expiration time of the search to the current time (now) plus - the time-to-live (ttl) value. - - :return: The :class:`Job`. - """ - self.post("control", action="touch") - return self - - def set_ttl(self, value): - """Set the job's time-to-live (ttl) value, which is the time before the - search job expires and is still available. - - :param `value`: The ttl value, in seconds. - :type value: ``integer`` - - :return: The :class:`Job`. - """ - self.post("control", action="setttl", ttl=value) - return self - - def unpause(self): - """Resumes the current search, if paused. - - :return: The :class:`Job`. - """ - self.post("control", action="unpause") - return self - - -class Jobs(Collection): - """This class represents a collection of search jobs. Retrieve this - collection using :meth:`Service.jobs`.""" - - def __init__(self, service): - # Splunk 9 introduces the v2 endpoint - if not service.disable_v2_api: - path = PATH_JOBS_V2 - else: - path = PATH_JOBS - Collection.__init__(self, service, path, item=Job) - # The count value to say list all the contents of this - # Collection is 0, not -1 as it is on most. - self.null_count = 0 - - def _load_list(self, response): - # Overridden because Job takes a sid instead of a path. - entries = _load_atom_entries(response) - if entries is None: return [] - entities = [] - for entry in entries: - state = _parse_atom_entry(entry) - entity = self.item( - self.service, - entry['content']['sid'], - state=state) - entities.append(entity) - return entities - - def create(self, query, **kwargs): - """ Creates a search using a search query and any additional parameters - you provide. - - :param query: The search query. - :type query: ``string`` - :param kwargs: Additiona parameters (optional). For a list of available - parameters, see `Search job parameters - `_ - on Splunk Developer Portal. - :type kwargs: ``dict`` - - :return: The :class:`Job`. - """ - if kwargs.get("exec_mode", None) == "oneshot": - raise TypeError("Cannot specify exec_mode=oneshot; use the oneshot method instead.") - response = self.post(search=query, **kwargs) - sid = _load_sid(response, kwargs.get("output_mode", None)) - return Job(self.service, sid) - - def export(self, query, **params): - """Runs a search and immediately starts streaming preview events. This method returns a streaming handle to - this job's events as an XML document from the server. To parse this stream into usable Python objects, - pass the handle to :class:`splunklib.results.JSONResultsReader` along with the query param - "output_mode='json'":: - - import splunklib.client as client - import splunklib.results as results - service = client.connect(...) - rr = results.JSONResultsReader(service.jobs.export("search * | head 5",output_mode='json')) - for result in rr: - if isinstance(result, results.Message): - # Diagnostic messages may be returned in the results - print(f'{result.type}: {result.message}') - elif isinstance(result, dict): - # Normal events are returned as dicts - print(result) - assert rr.is_preview == False - - Running an export search is more efficient as it streams the results - directly to you, rather than having to write them out to disk and make - them available later. As soon as results are ready, you will receive - them. - - The ``export`` method makes a single roundtrip to the server (as opposed - to two for :meth:`create` followed by :meth:`preview`), plus at most two - more if the ``autologin`` field of :func:`connect` is set to ``True``. - - :raises `ValueError`: Raised for invalid queries. - :param query: The search query. - :type query: ``string`` - :param params: Additional arguments (optional). For a list of valid - parameters, see `GET search/jobs/export - `_ - in the REST API documentation. - :type params: ``dict`` - - :return: The ``InputStream`` IO handle to raw XML returned from the server. - """ - if "exec_mode" in params: - raise TypeError("Cannot specify an exec_mode to export.") - params['segmentation'] = params.get('segmentation', 'none') - return self.post(path_segment="export", - search=query, - **params).body - - def itemmeta(self): - """There is no metadata available for class:``Jobs``. - - Any call to this method raises a class:``NotSupportedError``. - - :raises: class:``NotSupportedError`` - """ - raise NotSupportedError() - - def oneshot(self, query, **params): - """Run a oneshot search and returns a streaming handle to the results. - - The ``InputStream`` object streams fragments from the server. To parse this stream into usable Python - objects, pass the handle to :class:`splunklib.results.JSONResultsReader` along with the query param - "output_mode='json'" :: - - import splunklib.client as client - import splunklib.results as results - service = client.connect(...) - rr = results.JSONResultsReader(service.jobs.oneshot("search * | head 5",output_mode='json')) - for result in rr: - if isinstance(result, results.Message): - # Diagnostic messages may be returned in the results - print(f'{result.type}: {result.message}') - elif isinstance(result, dict): - # Normal events are returned as dicts - print(result) - assert rr.is_preview == False - - The ``oneshot`` method makes a single roundtrip to the server (as opposed - to two for :meth:`create` followed by :meth:`results`), plus at most two more - if the ``autologin`` field of :func:`connect` is set to ``True``. - - :raises ValueError: Raised for invalid queries. - - :param query: The search query. - :type query: ``string`` - :param params: Additional arguments (optional): - - - "output_mode": Specifies the output format of the results (XML, - JSON, or CSV). - - - "earliest_time": Specifies the earliest time in the time range to - search. The time string can be a UTC time (with fractional seconds), - a relative time specifier (to now), or a formatted time string. - - - "latest_time": Specifies the latest time in the time range to - search. The time string can be a UTC time (with fractional seconds), - a relative time specifier (to now), or a formatted time string. - - - "rf": Specifies one or more fields to add to the search. - - :type params: ``dict`` - - :return: The ``InputStream`` IO handle to raw XML returned from the server. - """ - if "exec_mode" in params: - raise TypeError("Cannot specify an exec_mode to oneshot.") - params['segmentation'] = params.get('segmentation', 'none') - return self.post(search=query, - exec_mode="oneshot", - **params).body - - -class Loggers(Collection): - """This class represents a collection of service logging categories. - Retrieve this collection using :meth:`Service.loggers`.""" - - def __init__(self, service): - Collection.__init__(self, service, PATH_LOGGER) - - def itemmeta(self): - """There is no metadata available for class:``Loggers``. - - Any call to this method raises a class:``NotSupportedError``. - - :raises: class:``NotSupportedError`` - """ - raise NotSupportedError() - - -class Message(Entity): - def __init__(self, service, path, **kwargs): - Entity.__init__(self, service, path, **kwargs) - - @property - def value(self): - """Returns the message value. - - :return: The message value. - :rtype: ``string`` - """ - return self[self.name] - - -class ModularInputKind(Entity): - """This class contains the different types of modular inputs. Retrieve this - collection using :meth:`Service.modular_input_kinds`. - """ - - def __contains__(self, name): - args = self.state.content['endpoints']['args'] - if name in args: - return True - return Entity.__contains__(self, name) - - def __getitem__(self, name): - args = self.state.content['endpoint']['args'] - if name in args: - return args['item'] - return Entity.__getitem__(self, name) - - @property - def arguments(self): - """A dictionary of all the arguments supported by this modular input kind. - - The keys in the dictionary are the names of the arguments. The values are - another dictionary giving the metadata about that argument. The possible - keys in that dictionary are ``"title"``, ``"description"``, ``"required_on_create``", - ``"required_on_edit"``, ``"data_type"``. Each value is a string. It should be one - of ``"true"`` or ``"false"`` for ``"required_on_create"`` and ``"required_on_edit"``, - and one of ``"boolean"``, ``"string"``, or ``"number``" for ``"data_type"``. - - :return: A dictionary describing the arguments this modular input kind takes. - :rtype: ``dict`` - """ - return self.state.content['endpoint']['args'] - - def update(self, **kwargs): - """Raises an error. Modular input kinds are read only.""" - raise IllegalOperationException("Modular input kinds cannot be updated via the REST API.") - - -class SavedSearch(Entity): - """This class represents a saved search.""" - - def __init__(self, service, path, **kwargs): - Entity.__init__(self, service, path, **kwargs) - - def acknowledge(self): - """Acknowledges the suppression of alerts from this saved search and - resumes alerting. - - :return: The :class:`SavedSearch`. - """ - self.post("acknowledge") - return self - - @property - def alert_count(self): - """Returns the number of alerts fired by this saved search. - - :return: The number of alerts fired by this saved search. - :rtype: ``integer`` - """ - return int(self._state.content.get('triggered_alert_count', 0)) - - def dispatch(self, **kwargs): - """Runs the saved search and returns the resulting search job. - - :param `kwargs`: Additional dispatch arguments (optional). For details, - see the `POST saved/searches/{name}/dispatch - `_ - endpoint in the REST API documentation. - :type kwargs: ``dict`` - :return: The :class:`Job`. - """ - response = self.post("dispatch", **kwargs) - sid = _load_sid(response, kwargs.get("output_mode", None)) - return Job(self.service, sid) - - @property - def fired_alerts(self): - """Returns the collection of fired alerts (a fired alert group) - corresponding to this saved search's alerts. - - :raises IllegalOperationException: Raised when the search is not scheduled. - - :return: A collection of fired alerts. - :rtype: :class:`AlertGroup` - """ - if self['is_scheduled'] == '0': - raise IllegalOperationException('Unscheduled saved searches have no alerts.') - c = Collection( - self.service, - self.service._abspath(PATH_FIRED_ALERTS + self.name, - owner=self._state.access.owner, - app=self._state.access.app, - sharing=self._state.access.sharing), - item=AlertGroup) - return c - - def history(self, **kwargs): - """Returns a list of search jobs corresponding to this saved search. - - :param `kwargs`: Additional arguments (optional). - :type kwargs: ``dict`` - - :return: A list of :class:`Job` objects. - """ - response = self.get("history", **kwargs) - entries = _load_atom_entries(response) - if entries is None: return [] - jobs = [] - for entry in entries: - job = Job(self.service, entry.title) - jobs.append(job) - return jobs - - def update(self, search=None, **kwargs): - """Updates the server with any changes you've made to the current saved - search along with any additional arguments you specify. - - :param `search`: The search query (optional). - :type search: ``string`` - :param `kwargs`: Additional arguments (optional). For a list of available - parameters, see `Saved search parameters - `_ - on Splunk Developer Portal. - :type kwargs: ``dict`` - - :return: The :class:`SavedSearch`. - """ - # Updates to a saved search *require* that the search string be - # passed, so we pass the current search string if a value wasn't - # provided by the caller. - if search is None: search = self.content.search - Entity.update(self, search=search, **kwargs) - return self - - def scheduled_times(self, earliest_time='now', latest_time='+1h'): - """Returns the times when this search is scheduled to run. - - By default this method returns the times in the next hour. For different - time ranges, set *earliest_time* and *latest_time*. For example, - for all times in the last day use "earliest_time=-1d" and - "latest_time=now". - - :param earliest_time: The earliest time. - :type earliest_time: ``string`` - :param latest_time: The latest time. - :type latest_time: ``string`` - - :return: The list of search times. - """ - response = self.get("scheduled_times", - earliest_time=earliest_time, - latest_time=latest_time) - data = self._load_atom_entry(response) - rec = _parse_atom_entry(data) - times = [datetime.fromtimestamp(int(t)) - for t in rec.content.scheduled_times] - return times - - def suppress(self, expiration): - """Skips any scheduled runs of this search in the next *expiration* - number of seconds. - - :param expiration: The expiration period, in seconds. - :type expiration: ``integer`` - - :return: The :class:`SavedSearch`. - """ - self.post("suppress", expiration=expiration) - return self - - @property - def suppressed(self): - """Returns the number of seconds that this search is blocked from running - (possibly 0). - - :return: The number of seconds. - :rtype: ``integer`` - """ - r = self._run_action("suppress") - if r.suppressed == "1": - return int(r.expiration) - return 0 - - def unsuppress(self): - """Cancels suppression and makes this search run as scheduled. - - :return: The :class:`SavedSearch`. - """ - self.post("suppress", expiration="0") - return self - - -class SavedSearches(Collection): - """This class represents a collection of saved searches. Retrieve this - collection using :meth:`Service.saved_searches`.""" - - def __init__(self, service): - Collection.__init__( - self, service, PATH_SAVED_SEARCHES, item=SavedSearch) - - def create(self, name, search, **kwargs): - """ Creates a saved search. - - :param name: The name for the saved search. - :type name: ``string`` - :param search: The search query. - :type search: ``string`` - :param kwargs: Additional arguments (optional). For a list of available - parameters, see `Saved search parameters - `_ - on Splunk Developer Portal. - :type kwargs: ``dict`` - :return: The :class:`SavedSearches` collection. - """ - return Collection.create(self, name, search=search, **kwargs) - - -class Settings(Entity): - """This class represents configuration settings for a Splunk service. - Retrieve this collection using :meth:`Service.settings`.""" - - def __init__(self, service, **kwargs): - Entity.__init__(self, service, "/services/server/settings", **kwargs) - - # Updates on the settings endpoint are POSTed to server/settings/settings. - def update(self, **kwargs): - """Updates the settings on the server using the arguments you provide. - - :param kwargs: Additional arguments. For a list of valid arguments, see - `POST server/settings/{name} - `_ - in the REST API documentation. - :type kwargs: ``dict`` - :return: The :class:`Settings` collection. - """ - self.service.post("/services/server/settings/settings", **kwargs) - return self - - -class User(Entity): - """This class represents a Splunk user. - """ - - @property - def role_entities(self): - """Returns a list of roles assigned to this user. - - :return: The list of roles. - :rtype: ``list`` - """ - all_role_names = [r.name for r in self.service.roles.list()] - return [self.service.roles[name] for name in self.content.roles if name in all_role_names] - - -# Splunk automatically lowercases new user names so we need to match that -# behavior here to ensure that the subsequent member lookup works correctly. -class Users(Collection): - """This class represents the collection of Splunk users for this instance of - Splunk. Retrieve this collection using :meth:`Service.users`. - """ - - def __init__(self, service): - Collection.__init__(self, service, PATH_USERS, item=User) - - def __getitem__(self, key): - return Collection.__getitem__(self, key.lower()) - - def __contains__(self, name): - return Collection.__contains__(self, name.lower()) - - def create(self, username, password, roles, **params): - """Creates a new user. - - This function makes two roundtrips to the server, plus at most - two more if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param username: The username. - :type username: ``string`` - :param password: The password. - :type password: ``string`` - :param roles: A single role or list of roles for the user. - :type roles: ``string`` or ``list`` - :param params: Additional arguments (optional). For a list of available - parameters, see `User authentication parameters - `_ - on Splunk Developer Portal. - :type params: ``dict`` - - :return: The new user. - :rtype: :class:`User` - - **Example**:: - - import splunklib.client as client - c = client.connect(...) - users = c.users - boris = users.create("boris", "securepassword", roles="user") - hilda = users.create("hilda", "anotherpassword", roles=["user","power"]) - """ - if not isinstance(username, str): - raise ValueError(f"Invalid username: {str(username)}") - username = username.lower() - self.post(name=username, password=password, roles=roles, **params) - # splunkd doesn't return the user in the POST response body, - # so we have to make a second round trip to fetch it. - response = self.get(username) - entry = _load_atom(response, XNAME_ENTRY).entry - state = _parse_atom_entry(entry) - entity = self.item( - self.service, - parse.unquote(state.links.alternate), - state=state) - return entity - - def delete(self, name): - """ Deletes the user and returns the resulting collection of users. - - :param name: The name of the user to delete. - :type name: ``string`` - - :return: - :rtype: :class:`Users` - """ - return Collection.delete(self, name.lower()) - - -class Role(Entity): - """This class represents a user role. - """ - - def grant(self, *capabilities_to_grant): - """Grants additional capabilities to this role. - - :param capabilities_to_grant: Zero or more capabilities to grant this - role. For a list of capabilities, see - `Capabilities `_ - on Splunk Developer Portal. - :type capabilities_to_grant: ``string`` or ``list`` - :return: The :class:`Role`. - - **Example**:: - - service = client.connect(...) - role = service.roles['somerole'] - role.grant('change_own_password', 'search') - """ - possible_capabilities = self.service.capabilities - for capability in capabilities_to_grant: - if capability not in possible_capabilities: - raise NoSuchCapability(capability) - new_capabilities = self['capabilities'] + list(capabilities_to_grant) - self.post(capabilities=new_capabilities) - return self - - def revoke(self, *capabilities_to_revoke): - """Revokes zero or more capabilities from this role. - - :param capabilities_to_revoke: Zero or more capabilities to grant this - role. For a list of capabilities, see - `Capabilities `_ - on Splunk Developer Portal. - :type capabilities_to_revoke: ``string`` or ``list`` - - :return: The :class:`Role`. - - **Example**:: - - service = client.connect(...) - role = service.roles['somerole'] - role.revoke('change_own_password', 'search') - """ - possible_capabilities = self.service.capabilities - for capability in capabilities_to_revoke: - if capability not in possible_capabilities: - raise NoSuchCapability(capability) - old_capabilities = self['capabilities'] - new_capabilities = [] - for c in old_capabilities: - if c not in capabilities_to_revoke: - new_capabilities.append(c) - if not new_capabilities: - new_capabilities = '' # Empty lists don't get passed in the body, so we have to force an empty argument. - self.post(capabilities=new_capabilities) - return self - - -class Roles(Collection): - """This class represents the collection of roles in the Splunk instance. - Retrieve this collection using :meth:`Service.roles`.""" - - def __init__(self, service): - Collection.__init__(self, service, PATH_ROLES, item=Role) - - def __getitem__(self, key): - return Collection.__getitem__(self, key.lower()) - - def __contains__(self, name): - return Collection.__contains__(self, name.lower()) - - def create(self, name, **params): - """Creates a new role. - - This function makes two roundtrips to the server, plus at most - two more if - the ``autologin`` field of :func:`connect` is set to ``True``. - - :param name: Name for the role. - :type name: ``string`` - :param params: Additional arguments (optional). For a list of available - parameters, see `Roles parameters - `_ - on Splunk Developer Portal. - :type params: ``dict`` - - :return: The new role. - :rtype: :class:`Role` - - **Example**:: - - import splunklib.client as client - c = client.connect(...) - roles = c.roles - paltry = roles.create("paltry", imported_roles="user", defaultApp="search") - """ - if not isinstance(name, str): - raise ValueError(f"Invalid role name: {str(name)}") - name = name.lower() - self.post(name=name, **params) - # splunkd doesn't return the user in the POST response body, - # so we have to make a second round trip to fetch it. - response = self.get(name) - entry = _load_atom(response, XNAME_ENTRY).entry - state = _parse_atom_entry(entry) - entity = self.item( - self.service, - parse.unquote(state.links.alternate), - state=state) - return entity - - def delete(self, name): - """ Deletes the role and returns the resulting collection of roles. - - :param name: The name of the role to delete. - :type name: ``string`` - - :rtype: The :class:`Roles` - """ - return Collection.delete(self, name.lower()) - - -class Application(Entity): - """Represents a locally-installed Splunk app.""" - - @property - def setupInfo(self): - """Returns the setup information for the app. - - :return: The setup information. - """ - return self.content.get('eai:setup', None) - - def package(self): - """ Creates a compressed package of the app for archiving.""" - return self._run_action("package") - - def updateInfo(self): - """Returns any update information that is available for the app.""" - return self._run_action("update") - - -class KVStoreCollections(Collection): - def __init__(self, service): - Collection.__init__(self, service, 'storage/collections/config', item=KVStoreCollection) - - def __getitem__(self, item): - res = Collection.__getitem__(self, item) - for k, v in res.content.items(): - if "accelerated_fields" in k: - res.content[k] = json.loads(v) - return res - - def create(self, name, accelerated_fields={}, fields={}, **kwargs): - """Creates a KV Store Collection. - - :param name: name of collection to create - :type name: ``string`` - :param accelerated_fields: dictionary of accelerated_fields definitions - :type accelerated_fields: ``dict`` - :param fields: dictionary of field definitions - :type fields: ``dict`` - :param kwargs: a dictionary of additional parameters specifying indexes and field definitions - :type kwargs: ``dict`` - - :return: Result of POST request - """ - for k, v in accelerated_fields.items(): - if isinstance(v, dict): - v = json.dumps(v) - kwargs['accelerated_fields.' + k] = v - for k, v in fields.items(): - kwargs['field.' + k] = v - return self.post(name=name, **kwargs) - - -class KVStoreCollection(Entity): - @property - def data(self): - """Returns data object for this Collection. - - :rtype: :class:`KVStoreCollectionData` - """ - return KVStoreCollectionData(self) - - def update_accelerated_field(self, name, value): - """Changes the definition of a KV Store accelerated_field. - - :param name: name of accelerated_fields to change - :type name: ``string`` - :param value: new accelerated_fields definition - :type value: ``dict`` - - :return: Result of POST request - """ - kwargs = {} - kwargs['accelerated_fields.' + name] = json.dumps(value) if isinstance(value, dict) else value - return self.post(**kwargs) - - def update_field(self, name, value): - """Changes the definition of a KV Store field. - - :param name: name of field to change - :type name: ``string`` - :param value: new field definition - :type value: ``string`` - - :return: Result of POST request - """ - kwargs = {} - kwargs['field.' + name] = value - return self.post(**kwargs) - - -class KVStoreCollectionData: - """This class represents the data endpoint for a KVStoreCollection. - - Retrieve using :meth:`KVStoreCollection.data` - """ - JSON_HEADER = [('Content-Type', 'application/json')] - - def __init__(self, collection): - self.service = collection.service - self.collection = collection - self.owner, self.app, self.sharing = collection._proper_namespace() - self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name, encode_slash=True) + '/' - - def _get(self, url, **kwargs): - return self.service.get(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) - - def _post(self, url, **kwargs): - return self.service.post(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) - - def _delete(self, url, **kwargs): - return self.service.delete(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) - - def query(self, **query): - """ - Gets the results of query, with optional parameters sort, limit, skip, and fields. - - :param query: Optional parameters. Valid options are sort, limit, skip, and fields - :type query: ``dict`` - - :return: Array of documents retrieved by query. - :rtype: ``array`` - """ - - for key, value in query.items(): - if isinstance(query[key], dict): - query[key] = json.dumps(value) - - return json.loads(self._get('', **query).body.read().decode('utf-8')) - - def query_by_id(self, id): - """ - Returns object with _id = id. - - :param id: Value for ID. If not a string will be coerced to string. - :type id: ``string`` - - :return: Document with id - :rtype: ``dict`` - """ - return json.loads(self._get(UrlEncoded(str(id), encode_slash=True)).body.read().decode('utf-8')) - - def insert(self, data): - """ - Inserts item into this collection. An _id field will be generated if not assigned in the data. - - :param data: Document to insert - :type data: ``string`` - - :return: _id of inserted object - :rtype: ``dict`` - """ - if isinstance(data, dict): - data = json.dumps(data) - return json.loads( - self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) - - def delete(self, query=None): - """ - Deletes all data in collection if query is absent. Otherwise, deletes all data matched by query. - - :param query: Query to select documents to delete - :type query: ``string`` - - :return: Result of DELETE request - """ - return self._delete('', **({'query': query}) if query else {}) - - def delete_by_id(self, id): - """ - Deletes document that has _id = id. - - :param id: id of document to delete - :type id: ``string`` - - :return: Result of DELETE request - """ - return self._delete(UrlEncoded(str(id), encode_slash=True)) - - def update(self, id, data): - """ - Replaces document with _id = id with data. - - :param id: _id of document to update - :type id: ``string`` - :param data: the new document to insert - :type data: ``string`` - - :return: id of replaced document - :rtype: ``dict`` - """ - if isinstance(data, dict): - data = json.dumps(data) - return json.loads(self._post(UrlEncoded(str(id), encode_slash=True), headers=KVStoreCollectionData.JSON_HEADER, - body=data).body.read().decode('utf-8')) - - def batch_find(self, *dbqueries): - """ - Returns array of results from queries dbqueries. - - :param dbqueries: Array of individual queries as dictionaries - :type dbqueries: ``array`` of ``dict`` - - :return: Results of each query - :rtype: ``array`` of ``array`` - """ - if len(dbqueries) < 1: - raise Exception('Must have at least one query.') - - data = json.dumps(dbqueries) - - return json.loads( - self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) - - def batch_save(self, *documents): - """ - Inserts or updates every document specified in documents. - - :param documents: Array of documents to save as dictionaries - :type documents: ``array`` of ``dict`` - - :return: Results of update operation as overall stats - :rtype: ``dict`` - """ - if len(documents) < 1: - raise Exception('Must have at least one document.') - - data = json.dumps(documents) - - return json.loads( - self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# The purpose of this module is to provide a friendlier domain interface to +# various Splunk endpoints. The approach here is to leverage the binding +# layer to capture endpoint context and provide objects and methods that +# offer simplified access their corresponding endpoints. The design avoids +# caching resource state. From the perspective of this module, the 'policy' +# for caching resource state belongs in the application or a higher level +# framework, and its the purpose of this module to provide simplified +# access to that resource state. +# +# A side note, the objects below that provide helper methods for updating eg: +# Entity state, are written so that they may be used in a fluent style. +# + +"""The **splunklib.client** module provides a Pythonic interface to the +`Splunk REST API `_, +allowing you programmatically access Splunk's resources. + +**splunklib.client** wraps a Pythonic layer around the wire-level +binding of the **splunklib.binding** module. The core of the library is the +:class:`Service` class, which encapsulates a connection to the server, and +provides access to the various aspects of Splunk's functionality, which are +exposed via the REST API. Typically you connect to a running Splunk instance +with the :func:`connect` function:: + + import splunklib.client as client + service = client.connect(host='localhost', port=8089, + username='admin', password='...') + assert isinstance(service, client.Service) + +:class:`Service` objects have fields for the various Splunk resources (such as apps, +jobs, saved searches, inputs, and indexes). All of these fields are +:class:`Collection` objects:: + + appcollection = service.apps + my_app = appcollection.create('my_app') + my_app = appcollection['my_app'] + appcollection.delete('my_app') + +The individual elements of the collection, in this case *applications*, +are subclasses of :class:`Entity`. An ``Entity`` object has fields for its +attributes, and methods that are specific to each kind of entity. For example:: + + print(my_app['author']) # Or: print(my_app.author) + my_app.package() # Creates a compressed package of this application +""" + +import contextlib +import datetime +import json +import logging +import re +import socket +from datetime import datetime, timedelta +from time import sleep +from urllib import parse + +from splunklib import data +from splunklib.data import record +from splunklib.binding import (AuthenticationError, Context, HTTPError, UrlEncoded, + _encode, _make_cookie_header, _NoAuthenticationToken, + namespace) + +logger = logging.getLogger(__name__) + +__all__ = [ + "connect", + "NotSupportedError", + "OperationError", + "IncomparableException", + "Service", + "namespace", + "AuthenticationError" +] + +PATH_APPS = "apps/local/" +PATH_CAPABILITIES = "authorization/capabilities/" +PATH_CONF = "configs/conf-%s/" +PATH_PROPERTIES = "properties/" +PATH_DEPLOYMENT_CLIENTS = "deployment/client/" +PATH_DEPLOYMENT_TENANTS = "deployment/tenants/" +PATH_DEPLOYMENT_SERVERS = "deployment/server/" +PATH_DEPLOYMENT_SERVERCLASSES = "deployment/serverclass/" +PATH_EVENT_TYPES = "saved/eventtypes/" +PATH_FIRED_ALERTS = "alerts/fired_alerts/" +PATH_INDEXES = "data/indexes/" +PATH_INPUTS = "data/inputs/" +PATH_JOBS = "search/jobs/" +PATH_JOBS_V2 = "search/v2/jobs/" +PATH_LOGGER = "/services/server/logger/" +PATH_MESSAGES = "messages/" +PATH_MODULAR_INPUTS = "data/modular-inputs" +PATH_ROLES = "authorization/roles/" +PATH_SAVED_SEARCHES = "saved/searches/" +PATH_STANZA = "configs/conf-%s/%s" # (file, stanza) +PATH_USERS = "authentication/users/" +PATH_RECEIVERS_STREAM = "/services/receivers/stream" +PATH_RECEIVERS_SIMPLE = "/services/receivers/simple" +PATH_STORAGE_PASSWORDS = "storage/passwords" + +XNAMEF_ATOM = "{http://www.w3.org/2005/Atom}%s" +XNAME_ENTRY = XNAMEF_ATOM % "entry" +XNAME_CONTENT = XNAMEF_ATOM % "content" + +MATCH_ENTRY_CONTENT = f"{XNAME_ENTRY}/{XNAME_CONTENT}/*" + + +class IllegalOperationException(Exception): + """Thrown when an operation is not possible on the Splunk instance that a + :class:`Service` object is connected to.""" + + +class IncomparableException(Exception): + """Thrown when trying to compare objects (using ``==``, ``<``, ``>``, and + so on) of a type that doesn't support it.""" + + +class AmbiguousReferenceException(ValueError): + """Thrown when the name used to fetch an entity matches more than one entity.""" + + +class InvalidNameException(Exception): + """Thrown when the specified name contains characters that are not allowed + in Splunk entity names.""" + + +class NoSuchCapability(Exception): + """Thrown when the capability that has been referred to doesn't exist.""" + + +class OperationError(Exception): + """Raised for a failed operation, such as a timeout.""" + + +class NotSupportedError(Exception): + """Raised for operations that are not supported on a given object.""" + + +def _trailing(template, *targets): + """Substring of *template* following all *targets*. + + **Example**:: + + template = "this is a test of the bunnies." + _trailing(template, "is", "est", "the") == " bunnies" + + Each target is matched successively in the string, and the string + remaining after the last target is returned. If one of the targets + fails to match, a ValueError is raised. + + :param template: Template to extract a trailing string from. + :type template: ``string`` + :param targets: Strings to successively match in *template*. + :type targets: list of ``string``s + :return: Trailing string after all targets are matched. + :rtype: ``string`` + :raises ValueError: Raised when one of the targets does not match. + """ + s = template + for t in targets: + n = s.find(t) + if n == -1: + raise ValueError("Target " + t + " not found in template.") + s = s[n + len(t):] + return s + + +# Filter the given state content record according to the given arg list. +def _filter_content(content, *args): + if len(args) > 0: + return record((k, content[k]) for k in args) + return record((k, v) for k, v in content.items() + if k not in ['eai:acl', 'eai:attributes', 'type']) + + +# Construct a resource path from the given base path + resource name +def _path(base, name): + if not base.endswith('/'): base = base + '/' + return base + name + + +# Load an atom record from the body of the given response +# this will ultimately be sent to an xml ElementTree so we +# should use the xmlcharrefreplace option +def _load_atom(response, match=None): + return data.load(response.body.read() + .decode('utf-8', 'xmlcharrefreplace'), match) + + +# Load an array of atom entries from the body of the given response +def _load_atom_entries(response): + r = _load_atom(response) + if 'feed' in r: + # Need this to handle a random case in the REST API + if r.feed.get('totalResults') in [0, '0']: + return [] + entries = r.feed.get('entry', None) + if entries is None: return None + return entries if isinstance(entries, list) else [entries] + # Unlike most other endpoints, the jobs endpoint does not return + # its state wrapped in another element, but at the top level. + # For example, in XML, it returns ... instead of + # .... + entries = r.get('entry', None) + if entries is None: return None + return entries if isinstance(entries, list) else [entries] + + +# Load the sid from the body of the given response +def _load_sid(response, output_mode): + if output_mode == "json": + json_obj = json.loads(response.body.read()) + return json_obj.get('sid') + return _load_atom(response).response.sid + + +# Parse the given atom entry record into a generic entity state record +def _parse_atom_entry(entry): + title = entry.get('title', None) + + elink = entry.get('link', []) + elink = elink if isinstance(elink, list) else [elink] + links = record((link.rel, link.href) for link in elink) + + # Retrieve entity content values + content = entry.get('content', {}) + + # Host entry metadata + metadata = _parse_atom_metadata(content) + + # Filter some of the noise out of the content record + content = record((k, v) for k, v in content.items() + if k not in ['eai:acl', 'eai:attributes']) + + if 'type' in content: + if isinstance(content['type'], list): + content['type'] = [t for t in content['type'] if t != 'text/xml'] + # Unset type if it was only 'text/xml' + if len(content['type']) == 0: + content.pop('type', None) + # Flatten 1 element list + if len(content['type']) == 1: + content['type'] = content['type'][0] + else: + content.pop('type', None) + + return record({ + 'title': title, + 'links': links, + 'access': metadata.access, + 'fields': metadata.fields, + 'content': content, + 'updated': entry.get("updated") + }) + + +# Parse the metadata fields out of the given atom entry content record +def _parse_atom_metadata(content): + # Hoist access metadata + access = content.get('eai:acl', None) + + # Hoist content metadata (and cleanup some naming) + attributes = content.get('eai:attributes', {}) + fields = record({ + 'required': attributes.get('requiredFields', []), + 'optional': attributes.get('optionalFields', []), + 'wildcard': attributes.get('wildcardFields', [])}) + + return record({'access': access, 'fields': fields}) + + +# kwargs: scheme, host, port, app, owner, username, password +def connect(**kwargs): + """This function connects and logs in to a Splunk instance. + + This function is a shorthand for :meth:`Service.login`. + The ``connect`` function makes one round trip to the server (for logging in). + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param verify: Enable (True) or disable (False) SSL verification for + https connections. (optional, the default is True) + :type verify: ``Boolean`` + :param `owner`: The owner context of the namespace (optional). + :type owner: ``string`` + :param `app`: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (the default is "user"). + :type sharing: "global", "system", "app", or "user" + :param `token`: The current session token (optional). Session tokens can be + shared across multiple service instances. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param autologin: When ``True``, automatically tries to log in again if the + session terminates. + :type autologin: ``boolean`` + :param `username`: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param `password`: The password for the Splunk account. + :type password: ``string`` + :param retires: Number of retries for each HTTP connection (optional, the default is 0). + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. + :type retries: ``int`` + :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryDelay: ``int`` (in seconds) + :param `context`: The SSLContext that can be used when setting verify=True (optional) + :type context: ``SSLContext`` + :return: An initialized :class:`Service` connection. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + a = s.apps["my_app"] + ... + """ + s = Service(**kwargs) + s.login() + return s + + +# In preparation for adding Storm support, we added an +# intermediary class between Service and Context. Storm's +# API is not going to be the same as enterprise Splunk's +# API, so we will derive both Service (for enterprise Splunk) +# and StormService for (Splunk Storm) from _BaseService, and +# put any shared behavior on it. +class _BaseService(Context): + pass + + +class Service(_BaseService): + """A Pythonic binding to Splunk instances. + + A :class:`Service` represents a binding to a Splunk instance on an + HTTP or HTTPS port. It handles the details of authentication, wire + formats, and wraps the REST API endpoints into something more + Pythonic. All of the low-level operations on the instance from + :class:`splunklib.binding.Context` are also available in case you need + to do something beyond what is provided by this class. + + After creating a ``Service`` object, you must call its :meth:`login` + method before you can issue requests to Splunk. + Alternately, use the :func:`connect` function to create an already + authenticated :class:`Service` object, or provide a session token + when creating the :class:`Service` object explicitly (the same + token may be shared by multiple :class:`Service` objects). + + :param host: The host name (the default is "localhost"). + :type host: ``string`` + :param port: The port number (the default is 8089). + :type port: ``integer`` + :param scheme: The scheme for accessing the service (the default is "https"). + :type scheme: "https" or "http" + :param verify: Enable (True) or disable (False) SSL verification for + https connections. (optional, the default is True) + :type verify: ``Boolean`` + :param `owner`: The owner context of the namespace (optional; use "-" for wildcard). + :type owner: ``string`` + :param `app`: The app context of the namespace (optional; use "-" for wildcard). + :type app: ``string`` + :param `token`: The current session token (optional). Session tokens can be + shared across multiple service instances. + :type token: ``string`` + :param cookie: A session cookie. When provided, you don't need to call :meth:`login`. + This parameter is only supported for Splunk 6.2+. + :type cookie: ``string`` + :param `username`: The Splunk account username, which is used to + authenticate the Splunk instance. + :type username: ``string`` + :param `password`: The password, which is used to authenticate the Splunk + instance. + :type password: ``string`` + :param retires: Number of retries for each HTTP connection (optional, the default is 0). + NOTE THAT THIS MAY INCREASE THE NUMBER OF ROUND TRIP CONNECTIONS TO THE SPLUNK SERVER. + :type retries: ``int`` + :param retryDelay: How long to wait between connection attempts if `retries` > 0 (optional, defaults to 10s). + :type retryDelay: ``int`` (in seconds) + :return: A :class:`Service` instance. + + **Example**:: + + import splunklib.client as client + s = client.Service(username="boris", password="natasha", ...) + s.login() + # Or equivalently + s = client.connect(username="boris", password="natasha") + # Or if you already have a session token + s = client.Service(token="atg232342aa34324a") + # Or if you already have a valid cookie + s = client.Service(cookie="splunkd_8089=...") + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._splunk_version = None + self._kvstore_owner = None + self._instance_type = None + + @property + def apps(self): + """Returns the collection of applications that are installed on this instance of Splunk. + + :return: A :class:`Collection` of :class:`Application` entities. + """ + return Collection(self, PATH_APPS, item=Application) + + @property + def confs(self): + """Returns the collection of configuration files for this Splunk instance. + + :return: A :class:`Configurations` collection of + :class:`ConfigurationFile` entities. + """ + return Configurations(self) + + @property + def capabilities(self): + """Returns the list of system capabilities. + + :return: A ``list`` of capabilities. + """ + response = self.get(PATH_CAPABILITIES) + return _load_atom(response, MATCH_ENTRY_CONTENT).capabilities + + @property + def event_types(self): + """Returns the collection of event types defined in this Splunk instance. + + :return: An :class:`Entity` containing the event types. + """ + return Collection(self, PATH_EVENT_TYPES) + + @property + def fired_alerts(self): + """Returns the collection of alerts that have been fired on the Splunk + instance, grouped by saved search. + + :return: A :class:`Collection` of :class:`AlertGroup` entities. + """ + return Collection(self, PATH_FIRED_ALERTS, item=AlertGroup) + + @property + def indexes(self): + """Returns the collection of indexes for this Splunk instance. + + :return: An :class:`Indexes` collection of :class:`Index` entities. + """ + return Indexes(self, PATH_INDEXES, item=Index) + + @property + def info(self): + """Returns the information about this instance of Splunk. + + :return: The system information, as key-value pairs. + :rtype: ``dict`` + """ + response = self.get("/services/server/info") + return _filter_content(_load_atom(response, MATCH_ENTRY_CONTENT)) + + def input(self, path, kind=None): + """Retrieves an input by path, and optionally kind. + + :return: A :class:`Input` object. + """ + return Input(self, path, kind=kind).refresh() + + @property + def inputs(self): + """Returns the collection of inputs configured on this Splunk instance. + + :return: An :class:`Inputs` collection of :class:`Input` entities. + """ + return Inputs(self) + + def job(self, sid): + """Retrieves a search job by sid. + + :return: A :class:`Job` object. + """ + return Job(self, sid).refresh() + + @property + def jobs(self): + """Returns the collection of current search jobs. + + :return: A :class:`Jobs` collection of :class:`Job` entities. + """ + return Jobs(self) + + @property + def loggers(self): + """Returns the collection of logging level categories and their status. + + :return: A :class:`Loggers` collection of logging levels. + """ + return Loggers(self) + + @property + def messages(self): + """Returns the collection of service messages. + + :return: A :class:`Collection` of :class:`Message` entities. + """ + return Collection(self, PATH_MESSAGES, item=Message) + + @property + def modular_input_kinds(self): + """Returns the collection of the modular input kinds on this Splunk instance. + + :return: A :class:`ReadOnlyCollection` of :class:`ModularInputKind` entities. + """ + if self.splunk_version >= (5,): + return ReadOnlyCollection(self, PATH_MODULAR_INPUTS, item=ModularInputKind) + raise IllegalOperationException("Modular inputs are not supported before Splunk version 5.") + + @property + def storage_passwords(self): + """Returns the collection of the storage passwords on this Splunk instance. + + :return: A :class:`ReadOnlyCollection` of :class:`StoragePasswords` entities. + """ + return StoragePasswords(self) + + # kwargs: enable_lookups, reload_macros, parse_only, output_mode + def parse(self, query, **kwargs): + """Parses a search query and returns a semantic map of the search. + + :param query: The search query to parse. + :type query: ``string`` + :param kwargs: Arguments to pass to the ``search/parser`` endpoint + (optional). Valid arguments are: + + * "enable_lookups" (``boolean``): If ``True``, performs reverse lookups + to expand the search expression. + + * "output_mode" (``string``): The output format (XML or JSON). + + * "parse_only" (``boolean``): If ``True``, disables the expansion of + search due to evaluation of subsearches, time term expansion, + lookups, tags, eventtypes, and sourcetype alias. + + * "reload_macros" (``boolean``): If ``True``, reloads macro + definitions from macros.conf. + + :type kwargs: ``dict`` + :return: A semantic map of the parsed search query. + """ + if not self.disable_v2_api: + return self.post("search/v2/parser", q=query, **kwargs) + return self.get("search/parser", q=query, **kwargs) + + def restart(self, timeout=None): + """Restarts this Splunk instance. + + The service is unavailable until it has successfully restarted. + + If a *timeout* value is specified, ``restart`` blocks until the service + resumes or the timeout period has been exceeded. Otherwise, ``restart`` returns + immediately. + + :param timeout: A timeout period, in seconds. + :type timeout: ``integer`` + """ + msg = {"value": "Restart requested by " + self.username + "via the Splunk SDK for Python"} + # This message will be deleted once the server actually restarts. + self.messages.create(name="restart_required", **msg) + result = self.post("/services/server/control/restart") + if timeout is None: + return result + start = datetime.now() + diff = timedelta(seconds=timeout) + while datetime.now() - start < diff: + try: + self.login() + if not self.restart_required: + return result + except Exception as e: + sleep(1) + raise Exception("Operation time out.") + + @property + def restart_required(self): + """Indicates whether splunkd is in a state that requires a restart. + + :return: A ``boolean`` that indicates whether a restart is required. + + """ + response = self.get("messages").body.read() + messages = data.load(response)['feed'] + if 'entry' not in messages: + result = False + else: + if isinstance(messages['entry'], dict): + titles = [messages['entry']['title']] + else: + titles = [x['title'] for x in messages['entry']] + result = 'restart_required' in titles + return result + + @property + def roles(self): + """Returns the collection of user roles. + + :return: A :class:`Roles` collection of :class:`Role` entities. + """ + return Roles(self) + + def search(self, query, **kwargs): + """Runs a search using a search query and any optional arguments you + provide, and returns a `Job` object representing the search. + + :param query: A search query. + :type query: ``string`` + :param kwargs: Arguments for the search (optional): + + * "output_mode" (``string``): Specifies the output format of the + results. + + * "earliest_time" (``string``): Specifies the earliest time in the + time range to + search. The time string can be a UTC time (with fractional + seconds), a relative time specifier (to now), or a formatted + time string. + + * "latest_time" (``string``): Specifies the latest time in the time + range to + search. The time string can be a UTC time (with fractional + seconds), a relative time specifier (to now), or a formatted + time string. + + * "rf" (``string``): Specifies one or more fields to add to the + search. + + :type kwargs: ``dict`` + :rtype: class:`Job` + :returns: An object representing the created job. + """ + return self.jobs.create(query, **kwargs) + + @property + def saved_searches(self): + """Returns the collection of saved searches. + + :return: A :class:`SavedSearches` collection of :class:`SavedSearch` + entities. + """ + return SavedSearches(self) + + @property + def settings(self): + """Returns the configuration settings for this instance of Splunk. + + :return: A :class:`Settings` object containing configuration settings. + """ + return Settings(self) + + @property + def splunk_version(self): + """Returns the version of the splunkd instance this object is attached + to. + + The version is returned as a tuple of the version components as + integers (for example, `(4,3,3)` or `(5,)`). + + :return: A ``tuple`` of ``integers``. + """ + if self._splunk_version is None: + self._splunk_version = tuple(int(p) for p in self.info['version'].split('.')) + return self._splunk_version + + @property + def splunk_instance(self): + if self._instance_type is None : + splunk_info = self.info + if hasattr(splunk_info, 'instance_type') : + self._instance_type = splunk_info['instance_type'] + else: + self._instance_type = '' + return self._instance_type + + @property + def disable_v2_api(self): + if self.splunk_instance.lower() == 'cloud': + return self.splunk_version < (9,0,2209) + return self.splunk_version < (9,0,2) + + @property + def kvstore_owner(self): + """Returns the KVStore owner for this instance of Splunk. + + By default is the kvstore owner is not set, it will return "nobody" + :return: A string with the KVStore owner. + """ + if self._kvstore_owner is None: + self._kvstore_owner = "nobody" + return self._kvstore_owner + + @kvstore_owner.setter + def kvstore_owner(self, value): + """ + kvstore is refreshed, when the owner value is changed + """ + self._kvstore_owner = value + self.kvstore + + @property + def kvstore(self): + """Returns the collection of KV Store collections. + + sets the owner for the namespace, before retrieving the KVStore Collection + + :return: A :class:`KVStoreCollections` collection of :class:`KVStoreCollection` entities. + """ + self.namespace['owner'] = self.kvstore_owner + return KVStoreCollections(self) + + @property + def users(self): + """Returns the collection of users. + + :return: A :class:`Users` collection of :class:`User` entities. + """ + return Users(self) + + +class Endpoint: + """This class represents individual Splunk resources in the Splunk REST API. + + An ``Endpoint`` object represents a URI, such as ``/services/saved/searches``. + This class provides the common functionality of :class:`Collection` and + :class:`Entity` (essentially HTTP GET and POST methods). + """ + + def __init__(self, service, path): + self.service = service + self.path = path + + def get_api_version(self, path): + """Return the API version of the service used in the provided path. + + Args: + path (str): A fully-qualified endpoint path (for example, "/services/search/jobs"). + + Returns: + int: Version of the API (for example, 1) + """ + # Default to v1 if undefined in the path + # For example, "/services/search/jobs" is using API v1 + api_version = 1 + + versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path) + if versionSearch: + api_version = int(versionSearch.group(1)) + + return api_version + + def get(self, path_segment="", owner=None, app=None, sharing=None, **query): + """Performs a GET operation on the path segment relative to this endpoint. + + This method is named to match the HTTP method. This method makes at least + one roundtrip to the server, one additional round trip for + each 303 status returned, plus at most two additional round + trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method takes a + default namespace from the :class:`Service` object for this :class:`Endpoint`. + All other keyword arguments are included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Service`` is not logged in. + :raises HTTPError: Raised when an error in the request occurs. + :param path_segment: A path segment relative to this endpoint. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (optional). + :type sharing: "global", "system", "app", or "user" + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + import splunklib.client + s = client.service(...) + apps = s.apps + apps.get() == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '26208'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:30:35 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + apps.get('nonexistant/path') # raises HTTPError + s.logout() + apps.get() # raises AuthenticationError + """ + # self.path to the Endpoint is relative in the SDK, so passing + # owner, app, sharing, etc. along will produce the correct + # namespace in the final request. + if path_segment.startswith('/'): + path = path_segment + else: + if not self.path.endswith('/') and path_segment != "": + self.path = self.path + '/' + path = self.service._abspath(self.path + path_segment, owner=owner, + app=app, sharing=sharing) + # ^-- This was "%s%s" % (self.path, path_segment). + # That doesn't work, because self.path may be UrlEncoded. + + # Get the API version from the path + api_version = self.get_api_version(path) + + # Search API v2+ fallback to v1: + # - In v2+, /results_preview, /events and /results do not support search params. + # - Fallback from v2+ to v1 if Splunk Version is < 9. + # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): + # path = path.replace(PATH_JOBS_V2, PATH_JOBS) + + if api_version == 1: + if isinstance(path, UrlEncoded): + path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) + else: + path = path.replace(PATH_JOBS_V2, PATH_JOBS) + + return self.service.get(path, + owner=owner, app=app, sharing=sharing, + **query) + + def post(self, path_segment="", owner=None, app=None, sharing=None, **query): + """Performs a POST operation on the path segment relative to this endpoint. + + This method is named to match the HTTP method. This method makes at least + one roundtrip to the server, one additional round trip for + each 303 status returned, plus at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + If *owner*, *app*, and *sharing* are omitted, this method takes a + default namespace from the :class:`Service` object for this :class:`Endpoint`. + All other keyword arguments are included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Service`` is not logged in. + :raises HTTPError: Raised when an error in the request occurs. + :param path_segment: A path segment relative to this endpoint. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode of the namespace (optional). + :type sharing: ``string`` + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + import splunklib.client + s = client.service(...) + apps = s.apps + apps.post(name='boris') == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '2908'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 18:34:50 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'Created', + 'status': 201} + apps.get('nonexistant/path') # raises HTTPError + s.logout() + apps.get() # raises AuthenticationError + """ + if path_segment.startswith('/'): + path = path_segment + else: + if not self.path.endswith('/') and path_segment != "": + self.path = self.path + '/' + path = self.service._abspath(self.path + path_segment, owner=owner, app=app, sharing=sharing) + + # Get the API version from the path + api_version = self.get_api_version(path) + + # Search API v2+ fallback to v1: + # - In v2+, /results_preview, /events and /results do not support search params. + # - Fallback from v2+ to v1 if Splunk Version is < 9. + # if api_version >= 2 and ('search' in query and path.endswith(tuple(["results_preview", "events", "results"])) or self.service.splunk_version < (9,)): + # path = path.replace(PATH_JOBS_V2, PATH_JOBS) + + if api_version == 1: + if isinstance(path, UrlEncoded): + path = UrlEncoded(path.replace(PATH_JOBS_V2, PATH_JOBS), skip_encode=True) + else: + path = path.replace(PATH_JOBS_V2, PATH_JOBS) + + return self.service.post(path, owner=owner, app=app, sharing=sharing, **query) + + +# kwargs: path, app, owner, sharing, state +class Entity(Endpoint): + """This class is a base class for Splunk entities in the REST API, such as + saved searches, jobs, indexes, and inputs. + + ``Entity`` provides the majority of functionality required by entities. + Subclasses only implement the special cases for individual entities. + For example for saved searches, the subclass makes fields like ``action.email``, + ``alert_type``, and ``search`` available. + + An ``Entity`` is addressed like a dictionary, with a few extensions, + so the following all work, for example in saved searches:: + + ent['action.email'] + ent['alert_type'] + ent['search'] + + You can also access the fields as though they were the fields of a Python + object, as in:: + + ent.alert_type + ent.search + + However, because some of the field names are not valid Python identifiers, + the dictionary-like syntax is preferable. + + The state of an :class:`Entity` object is cached, so accessing a field + does not contact the server. If you think the values on the + server have changed, call the :meth:`Entity.refresh` method. + """ + # Not every endpoint in the API is an Entity or a Collection. For + # example, a saved search at saved/searches/{name} has an additional + # method saved/searches/{name}/scheduled_times, but this isn't an + # entity in its own right. In these cases, subclasses should + # implement a method that uses the get and post methods inherited + # from Endpoint, calls the _load_atom function (it's elsewhere in + # client.py, but not a method of any object) to read the + # information, and returns the extracted data in a Pythonesque form. + # + # The primary use of subclasses of Entity is to handle specially + # named fields in the Entity. If you only need to provide a default + # value for an optional field, subclass Entity and define a + # dictionary ``defaults``. For instance,:: + # + # class Hypothetical(Entity): + # defaults = {'anOptionalField': 'foo', + # 'anotherField': 'bar'} + # + # If you have to do more than provide a default, such as rename or + # actually process values, then define a new method with the + # ``@property`` decorator. + # + # class Hypothetical(Entity): + # @property + # def foobar(self): + # return self.content['foo'] + "-" + self.content["bar"] + + # Subclasses can override defaults the default values for + # optional fields. See above. + defaults = {} + + def __init__(self, service, path, **kwargs): + Endpoint.__init__(self, service, path) + self._state = None + if not kwargs.get('skip_refresh', False): + self.refresh(kwargs.get('state', None)) # "Prefresh" + + def __contains__(self, item): + try: + self[item] + return True + except (KeyError, AttributeError): + return False + + def __eq__(self, other): + """Raises IncomparableException. + + Since Entity objects are snapshots of times on the server, no + simple definition of equality will suffice beyond instance + equality, and instance equality leads to strange situations + such as:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + x = saved_searches['asearch'] + + but then ``x != saved_searches['asearch']``. + + whether or not there was a change on the server. Rather than + try to do something fancy, we simply declare that equality is + undefined for Entities. + + Makes no roundtrips to the server. + """ + raise IncomparableException(f"Equality is undefined for objects of class {self.__class__.__name__}") + + def __getattr__(self, key): + # Called when an attribute was not found by the normal method. In this + # case we try to find it in self.content and then self.defaults. + if key in self.state.content: + return self.state.content[key] + if key in self.defaults: + return self.defaults[key] + raise AttributeError(key) + + def __getitem__(self, key): + # getattr attempts to find a field on the object in the normal way, + # then calls __getattr__ if it cannot. + return getattr(self, key) + + # Load the Atom entry record from the given response - this is a method + # because the "entry" record varies slightly by entity and this allows + # for a subclass to override and handle any special cases. + def _load_atom_entry(self, response): + elem = _load_atom(response, XNAME_ENTRY) + if isinstance(elem, list): + apps = [ele.entry.content.get('eai:appName') for ele in elem] + + raise AmbiguousReferenceException( + f"Fetch from server returned multiple entries for name '{elem[0].entry.title}' in apps {apps}.") + return elem.entry + + # Load the entity state record from the given response + def _load_state(self, response): + entry = self._load_atom_entry(response) + return _parse_atom_entry(entry) + + def _run_action(self, path_segment, **kwargs): + """Run a method and return the content Record from the returned XML. + + A method is a relative path from an Entity that is not itself + an Entity. _run_action assumes that the returned XML is an + Atom field containing one Entry, and the contents of Entry is + what should be the return value. This is right in enough cases + to make this method useful. + """ + response = self.get(path_segment, **kwargs) + data = self._load_atom_entry(response) + rec = _parse_atom_entry(data) + return rec.content + + def _proper_namespace(self, owner=None, app=None, sharing=None): + """Produce a namespace sans wildcards for use in entity requests. + + This method tries to fill in the fields of the namespace which are `None` + or wildcard (`'-'`) from the entity's namespace. If that fails, it uses + the service's namespace. + + :param owner: + :param app: + :param sharing: + :return: + """ + if owner is None and app is None and sharing is None: # No namespace provided + if self._state is not None and 'access' in self._state: + return (self._state.access.owner, + self._state.access.app, + self._state.access.sharing) + return (self.service.namespace['owner'], + self.service.namespace['app'], + self.service.namespace['sharing']) + return owner, app, sharing + + def delete(self): + owner, app, sharing = self._proper_namespace() + return self.service.delete(self.path, owner=owner, app=app, sharing=sharing) + + def get(self, path_segment="", owner=None, app=None, sharing=None, **query): + owner, app, sharing = self._proper_namespace(owner, app, sharing) + return super().get(path_segment, owner=owner, app=app, sharing=sharing, **query) + + def post(self, path_segment="", owner=None, app=None, sharing=None, **query): + owner, app, sharing = self._proper_namespace(owner, app, sharing) + return super().post(path_segment, owner=owner, app=app, sharing=sharing, **query) + + def refresh(self, state=None): + """Refreshes the state of this entity. + + If *state* is provided, load it as the new state for this + entity. Otherwise, make a roundtrip to the server (by calling + the :meth:`read` method of ``self``) to fetch an updated state, + plus at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param state: Entity-specific arguments (optional). + :type state: ``dict`` + :raises EntityDeletedException: Raised if the entity no longer exists on + the server. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + search = s.apps['search'] + search.refresh() + """ + if state is not None: + self._state = state + else: + self._state = self.read(self.get()) + return self + + @property + def access(self): + """Returns the access metadata for this entity. + + :return: A :class:`splunklib.data.Record` object with three keys: + ``owner``, ``app``, and ``sharing``. + """ + return self.state.access + + @property + def content(self): + """Returns the contents of the entity. + + :return: A ``dict`` containing values. + """ + return self.state.content + + def disable(self): + """Disables the entity at this endpoint.""" + self.post("disable") + return self + + def enable(self): + """Enables the entity at this endpoint.""" + self.post("enable") + return self + + @property + def fields(self): + """Returns the content metadata for this entity. + + :return: A :class:`splunklib.data.Record` object with three keys: + ``required``, ``optional``, and ``wildcard``. + """ + return self.state.fields + + @property + def links(self): + """Returns a dictionary of related resources. + + :return: A ``dict`` with keys and corresponding URLs. + """ + return self.state.links + + @property + def name(self): + """Returns the entity name. + + :return: The entity name. + :rtype: ``string`` + """ + return self.state.title + + def read(self, response): + """ Reads the current state of the entity from the server. """ + results = self._load_state(response) + # In lower layers of the SDK, we end up trying to URL encode + # text to be dispatched via HTTP. However, these links are already + # URL encoded when they arrive, and we need to mark them as such. + unquoted_links = dict((k, UrlEncoded(v, skip_encode=True)) + for k, v in results['links'].items()) + results['links'] = unquoted_links + return results + + def reload(self): + """Reloads the entity.""" + self.post("_reload") + return self + + def acl_update(self, **kwargs): + """To update Access Control List (ACL) properties for an endpoint. + + :param kwargs: Additional entity-specific arguments (required). + + - "owner" (``string``): The Splunk username, such as "admin". A value of "nobody" means no specific user (required). + + - "sharing" (``string``): A mode that indicates how the resource is shared. The sharing mode can be "user", "app", "global", or "system" (required). + + :type kwargs: ``dict`` + + **Example**:: + + import splunklib.client as client + service = client.connect(...) + saved_search = service.saved_searches["name"] + saved_search.acl_update(sharing="app", owner="nobody", app="search", **{"perms.read": "admin, nobody"}) + """ + if "body" not in kwargs: + kwargs = {"body": kwargs} + + if "sharing" not in kwargs["body"]: + raise ValueError("Required argument 'sharing' is missing.") + if "owner" not in kwargs["body"]: + raise ValueError("Required argument 'owner' is missing.") + + self.post("acl", **kwargs) + self.refresh() + return self + + @property + def state(self): + """Returns the entity's state record. + + :return: A ``dict`` containing fields and metadata for the entity. + """ + if self._state is None: self.refresh() + return self._state + + def update(self, **kwargs): + """Updates the server with any changes you've made to the current entity + along with any additional arguments you specify. + + **Note**: You cannot update the ``name`` field of an entity. + + Many of the fields in the REST API are not valid Python + identifiers, which means you cannot pass them as keyword + arguments. That is, Python will fail to parse the following:: + + # This fails + x.update(check-new=False, email.to='boris@utopia.net') + + However, you can always explicitly use a dictionary to pass + such keys:: + + # This works + x.update(**{'check-new': False, 'email.to': 'boris@utopia.net'}) + + :param kwargs: Additional entity-specific arguments (optional). + :type kwargs: ``dict`` + + :return: The entity this method is called on. + :rtype: class:`Entity` + """ + # The peculiarity in question: the REST API creates a new + # Entity if we pass name in the dictionary, instead of the + # expected behavior of updating this Entity. Therefore, we + # check for 'name' in kwargs and throw an error if it is + # there. + if 'name' in kwargs: + raise IllegalOperationException('Cannot update the name of an Entity via the REST API.') + self.post(**kwargs) + return self + + +class ReadOnlyCollection(Endpoint): + """This class represents a read-only collection of entities in the Splunk + instance. + """ + + def __init__(self, service, path, item=Entity): + Endpoint.__init__(self, service, path) + self.item = item # Item accessor + self.null_count = -1 + + def __contains__(self, name): + """Is there at least one entry called *name* in this collection? + + Makes a single roundtrip to the server, plus at most two more + if + the ``autologin`` field of :func:`connect` is set to ``True``. + """ + try: + self[name] + return True + except KeyError: + return False + except AmbiguousReferenceException: + return True + + def __getitem__(self, key): + """Fetch an item named *key* from this collection. + + A name is not a unique identifier in a collection. The unique + identifier is a name plus a namespace. For example, there can + be a saved search named ``'mysearch'`` with sharing ``'app'`` + in application ``'search'``, and another with sharing + ``'user'`` with owner ``'boris'`` and application + ``'search'``. If the ``Collection`` is attached to a + ``Service`` that has ``'-'`` (wildcard) as user and app in its + namespace, then both of these may be visible under the same + name. + + Where there is no conflict, ``__getitem__`` will fetch the + entity given just the name. If there is a conflict, and you + pass just a name, it will raise a ``ValueError``. In that + case, add the namespace as a second argument. + + This function makes a single roundtrip to the server, plus at + most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param key: The name to fetch, or a tuple (name, namespace). + :return: An :class:`Entity` object. + :raises KeyError: Raised if *key* does not exist. + :raises ValueError: Raised if no namespace is specified and *key* + does not refer to a unique name. + + **Example**:: + + s = client.connect(...) + saved_searches = s.saved_searches + x1 = saved_searches.create( + 'mysearch', 'search * | head 1', + owner='admin', app='search', sharing='app') + x2 = saved_searches.create( + 'mysearch', 'search * | head 1', + owner='admin', app='search', sharing='user') + # Raises ValueError: + saved_searches['mysearch'] + # Fetches x1 + saved_searches[ + 'mysearch', + client.namespace(sharing='app', app='search')] + # Fetches x2 + saved_searches[ + 'mysearch', + client.namespace(sharing='user', owner='boris', app='search')] + """ + try: + if isinstance(key, tuple) and len(key) == 2: + # x[a,b] is translated to x.__getitem__( (a,b) ), so we + # have to extract values out. + key, ns = key + key = UrlEncoded(key, encode_slash=True) + response = self.get(key, owner=ns.owner, app=ns.app) + else: + key = UrlEncoded(key, encode_slash=True) + response = self.get(key) + entries = self._load_list(response) + if len(entries) > 1: + raise AmbiguousReferenceException( + f"Found multiple entities named '{key}'; please specify a namespace.") + if len(entries) == 0: + raise KeyError(key) + return entries[0] + except HTTPError as he: + if he.status == 404: # No entity matching key and namespace. + raise KeyError(key) + else: + raise + + def __iter__(self, **kwargs): + """Iterate over the entities in the collection. + + :param kwargs: Additional arguments. + :type kwargs: ``dict`` + :rtype: iterator over entities. + + Implemented to give Collection a listish interface. This + function always makes a roundtrip to the server, plus at most + two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + for entity in saved_searches: + print(f"Saved search named {entity.name}") + """ + + for item in self.iter(**kwargs): + yield item + + def __len__(self): + """Enable ``len(...)`` for ``Collection`` objects. + + Implemented for consistency with a listish interface. No + further failure modes beyond those possible for any method on + an Endpoint. + + This function always makes a round trip to the server, plus at + most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + n = len(saved_searches) + """ + return len(self.list()) + + def _entity_path(self, state): + """Calculate the path to an entity to be returned. + + *state* should be the dictionary returned by + :func:`_parse_atom_entry`. :func:`_entity_path` extracts the + link to this entity from *state*, and strips all the namespace + prefixes from it to leave only the relative path of the entity + itself, sans namespace. + + :rtype: ``string`` + :return: an absolute path + """ + # This has been factored out so that it can be easily + # overloaded by Configurations, which has to switch its + # entities' endpoints from its own properties/ to configs/. + raw_path = parse.unquote(state.links.alternate) + if 'servicesNS/' in raw_path: + return _trailing(raw_path, 'servicesNS/', '/', '/') + if 'services/' in raw_path: + return _trailing(raw_path, 'services/') + return raw_path + + def _load_list(self, response): + """Converts *response* to a list of entities. + + *response* is assumed to be a :class:`Record` containing an + HTTP response, of the form:: + + {'status': 200, + 'headers': [('content-length', '232642'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Tue, 29 May 2012 15:27:08 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'body': ...a stream implementing .read()...} + + The ``'body'`` key refers to a stream containing an Atom feed, + that is, an XML document with a toplevel element ````, + and within that element one or more ```` elements. + """ + # Some subclasses of Collection have to override this because + # splunkd returns something that doesn't match + # . + entries = _load_atom_entries(response) + if entries is None: return [] + entities = [] + for entry in entries: + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + self._entity_path(state), + state=state) + entities.append(entity) + + return entities + + def itemmeta(self): + """Returns metadata for members of the collection. + + Makes a single roundtrip to the server, plus two more at most if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :return: A :class:`splunklib.data.Record` object containing the metadata. + + **Example**:: + + import splunklib.client as client + import pprint + s = client.connect(...) + pprint.pprint(s.apps.itemmeta()) + {'access': {'app': 'search', + 'can_change_perms': '1', + 'can_list': '1', + 'can_share_app': '1', + 'can_share_global': '1', + 'can_share_user': '1', + 'can_write': '1', + 'modifiable': '1', + 'owner': 'admin', + 'perms': {'read': ['*'], 'write': ['admin']}, + 'removable': '0', + 'sharing': 'user'}, + 'fields': {'optional': ['author', + 'configured', + 'description', + 'label', + 'manageable', + 'template', + 'visible'], + 'required': ['name'], 'wildcard': []}} + """ + response = self.get("_new") + content = _load_atom(response, MATCH_ENTRY_CONTENT) + return _parse_atom_metadata(content) + + def iter(self, offset=0, count=None, pagesize=None, **kwargs): + """Iterates over the collection. + + This method is equivalent to the :meth:`list` method, but + it returns an iterator and can load a certain number of entities at a + time from the server. + + :param offset: The index of the first entity to return (optional). + :type offset: ``integer`` + :param count: The maximum number of entities to return (optional). + :type count: ``integer`` + :param pagesize: The number of entities to load (optional). + :type pagesize: ``integer`` + :param kwargs: Additional arguments (optional): + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + for saved_search in s.saved_searches.iter(pagesize=10): + # Loads 10 saved searches at a time from the + # server. + ... + """ + assert pagesize is None or pagesize > 0 + if count is None: + count = self.null_count + fetched = 0 + while count == self.null_count or fetched < count: + response = self.get(count=pagesize or count, offset=offset, **kwargs) + items = self._load_list(response) + N = len(items) + fetched += N + for item in items: + yield item + if pagesize is None or N < pagesize: + break + offset += N + logger.debug("pagesize=%d, fetched=%d, offset=%d, N=%d, kwargs=%s", pagesize, fetched, offset, N, kwargs) + + # kwargs: count, offset, search, sort_dir, sort_key, sort_mode + def list(self, count=None, **kwargs): + """Retrieves a list of entities in this collection. + + The entire collection is loaded at once and is returned as a list. This + function makes a single roundtrip to the server, plus at most two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + There is no caching--every call makes at least one round trip. + + :param count: The maximum number of entities to return (optional). + :type count: ``integer`` + :param kwargs: Additional arguments (optional): + + - "offset" (``integer``): The offset of the first item to return. + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + :return: A ``list`` of entities. + """ + # response = self.get(count=count, **kwargs) + # return self._load_list(response) + return list(self.iter(count=count, **kwargs)) + + +class Collection(ReadOnlyCollection): + """A collection of entities. + + Splunk provides a number of different collections of distinct + entity types: applications, saved searches, fired alerts, and a + number of others. Each particular type is available separately + from the Splunk instance, and the entities of that type are + returned in a :class:`Collection`. + + The interface for :class:`Collection` does not quite match either + ``list`` or ``dict`` in Python, because there are enough semantic + mismatches with either to make its behavior surprising. A unique + element in a :class:`Collection` is defined by a string giving its + name plus namespace (although the namespace is optional if the name is + unique). + + **Example**:: + + import splunklib.client as client + service = client.connect(...) + mycollection = service.saved_searches + mysearch = mycollection['my_search', client.namespace(owner='boris', app='natasha', sharing='user')] + # Or if there is only one search visible named 'my_search' + mysearch = mycollection['my_search'] + + Similarly, ``name`` in ``mycollection`` works as you might expect (though + you cannot currently pass a namespace to the ``in`` operator), as does + ``len(mycollection)``. + + However, as an aggregate, :class:`Collection` behaves more like a + list. If you iterate over a :class:`Collection`, you get an + iterator over the entities, not the names and namespaces. + + **Example**:: + + for entity in mycollection: + assert isinstance(entity, client.Entity) + + Use the :meth:`create` and :meth:`delete` methods to create and delete + entities in this collection. To view the access control list and other + metadata of the collection, use the :meth:`ReadOnlyCollection.itemmeta` method. + + :class:`Collection` does no caching. Each call makes at least one + round trip to the server to fetch data. + """ + + def create(self, name, **params): + """Creates a new entity in this collection. + + This function makes either one or two roundtrips to the + server, depending on the type of entities in this + collection, plus at most two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param name: The name of the entity to create. + :type name: ``string`` + :param namespace: A namespace, as created by the :func:`splunklib.binding.namespace` + function (optional). You can also set ``owner``, ``app``, and + ``sharing`` in ``params``. + :type namespace: A :class:`splunklib.data.Record` object with keys ``owner``, ``app``, + and ``sharing``. + :param params: Additional entity-specific arguments (optional). + :type params: ``dict`` + :return: The new entity. + :rtype: A subclass of :class:`Entity`, chosen by :meth:`Collection.self.item`. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + applications = s.apps + new_app = applications.create("my_fake_app") + """ + if not isinstance(name, str): + raise InvalidNameException(f"{name} is not a valid name for an entity.") + if 'namespace' in params: + namespace = params.pop('namespace') + params['owner'] = namespace.owner + params['app'] = namespace.app + params['sharing'] = namespace.sharing + response = self.post(name=name, **params) + atom = _load_atom(response, XNAME_ENTRY) + if atom is None: + # This endpoint doesn't return the content of the new + # item. We have to go fetch it ourselves. + return self[name] + entry = atom.entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + self._entity_path(state), + state=state) + return entity + + def delete(self, name, **params): + """Deletes a specified entity from the collection. + + :param name: The name of the entity to delete. + :type name: ``string`` + :return: The collection. + :rtype: ``self`` + + This method is implemented for consistency with the REST API's DELETE + method. + + If there is no *name* entity on the server, a ``KeyError`` is + thrown. This function always makes a roundtrip to the server. + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + saved_searches = c.saved_searches + saved_searches.create('my_saved_search', + 'search * | head 1') + assert 'my_saved_search' in saved_searches + saved_searches.delete('my_saved_search') + assert 'my_saved_search' not in saved_searches + """ + name = UrlEncoded(name, encode_slash=True) + if 'namespace' in params: + namespace = params.pop('namespace') + params['owner'] = namespace.owner + params['app'] = namespace.app + params['sharing'] = namespace.sharing + try: + self.service.delete(_path(self.path, name), **params) + except HTTPError as he: + # An HTTPError with status code 404 means that the entity + # has already been deleted, and we reraise it as a + # KeyError. + if he.status == 404: + raise KeyError(f"No such entity {name}") + else: + raise + return self + + def get(self, name="", owner=None, app=None, sharing=None, **query): + """Performs a GET request to the server on the collection. + + If *owner*, *app*, and *sharing* are omitted, this method takes a + default namespace from the :class:`Service` object for this :class:`Endpoint`. + All other keyword arguments are included in the URL as query parameters. + + :raises AuthenticationError: Raised when the ``Service`` is not logged in. + :raises HTTPError: Raised when an error in the request occurs. + :param path_segment: A path segment relative to this endpoint. + :type path_segment: ``string`` + :param owner: The owner context of the namespace (optional). + :type owner: ``string`` + :param app: The app context of the namespace (optional). + :type app: ``string`` + :param sharing: The sharing mode for the namespace (optional). + :type sharing: "global", "system", "app", or "user" + :param query: All other keyword arguments, which are used as query + parameters. + :type query: ``string`` + :return: The response from the server. + :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``, + and ``status`` + + **Example**:: + + import splunklib.client + s = client.service(...) + saved_searches = s.saved_searches + saved_searches.get("my/saved/search") == \\ + {'body': ...a response reader object..., + 'headers': [('content-length', '26208'), + ('expires', 'Fri, 30 Oct 1998 00:00:00 GMT'), + ('server', 'Splunkd'), + ('connection', 'close'), + ('cache-control', 'no-store, max-age=0, must-revalidate, no-cache'), + ('date', 'Fri, 11 May 2012 16:30:35 GMT'), + ('content-type', 'text/xml; charset=utf-8')], + 'reason': 'OK', + 'status': 200} + saved_searches.get('nonexistant/search') # raises HTTPError + s.logout() + saved_searches.get() # raises AuthenticationError + + """ + name = UrlEncoded(name, encode_slash=True) + return super().get(name, owner, app, sharing, **query) + + +class ConfigurationFile(Collection): + """This class contains all of the stanzas from one configuration file. + """ + + # __init__'s arguments must match those of an Entity, not a + # Collection, since it is being created as the elements of a + # Configurations, which is a Collection subclass. + def __init__(self, service, path, **kwargs): + Collection.__init__(self, service, path, item=Stanza) + self.name = kwargs['state']['title'] + + +class Configurations(Collection): + """This class provides access to the configuration files from this Splunk + instance. Retrieve this collection using :meth:`Service.confs`. + + Splunk's configuration is divided into files, and each file into + stanzas. This collection is unusual in that the values in it are + themselves collections of :class:`ConfigurationFile` objects. + """ + + def __init__(self, service): + Collection.__init__(self, service, PATH_PROPERTIES, item=ConfigurationFile) + if self.service.namespace.owner == '-' or self.service.namespace.app == '-': + raise ValueError("Configurations cannot have wildcards in namespace.") + + def __getitem__(self, key): + # The superclass implementation is designed for collections that contain + # entities. This collection (Configurations) contains collections + # (ConfigurationFile). + # + # The configurations endpoint returns multiple entities when we ask for a single file. + # This screws up the default implementation of __getitem__ from Collection, which thinks + # that multiple entities means a name collision, so we have to override it here. + try: + self.get(key) + return ConfigurationFile(self.service, PATH_CONF % key, state={'title': key}) + except HTTPError as he: + if he.status == 404: # No entity matching key + raise KeyError(key) + else: + raise + + def __contains__(self, key): + # configs/conf-{name} never returns a 404. We have to post to properties/{name} + # in order to find out if a configuration exists. + try: + self.get(key) + return True + except HTTPError as he: + if he.status == 404: # No entity matching key + return False + raise + + def create(self, name): + """ Creates a configuration file named *name*. + + If there is already a configuration file with that name, + the existing file is returned. + + :param name: The name of the configuration file. + :type name: ``string`` + + :return: The :class:`ConfigurationFile` object. + """ + # This has to be overridden to handle the plumbing of creating + # a ConfigurationFile (which is a Collection) instead of some + # Entity. + if not isinstance(name, str): + raise ValueError(f"Invalid name: {repr(name)}") + response = self.post(__conf=name) + if response.status == 303: + return self[name] + if response.status == 201: + return ConfigurationFile(self.service, PATH_CONF % name, item=Stanza, state={'title': name}) + raise ValueError(f"Unexpected status code {response.status} returned from creating a stanza") + + def delete(self, key): + """Raises `IllegalOperationException`.""" + raise IllegalOperationException("Cannot delete configuration files from the REST API.") + + def _entity_path(self, state): + # Overridden to make all the ConfigurationFile objects + # returned refer to the configs/ path instead of the + # properties/ path used by Configrations. + return PATH_CONF % state['title'] + + +class Stanza(Entity): + """This class contains a single configuration stanza.""" + + def submit(self, stanza): + """Adds keys to the current configuration stanza as a + dictionary of key-value pairs. + + :param stanza: A dictionary of key-value pairs for the stanza. + :type stanza: ``dict`` + :return: The :class:`Stanza` object. + """ + body = _encode(**stanza) + self.service.post(self.path, body=body) + return self + + def __len__(self): + # The stanza endpoint returns all the keys at the same level in the XML as the eai information + # and 'disabled', so to get an accurate length, we have to filter those out and have just + # the stanza keys. + return len([x for x in self._state.content.keys() + if not x.startswith('eai') and x != 'disabled']) + + +class StoragePassword(Entity): + """This class contains a storage password. + """ + + def __init__(self, service, path, **kwargs): + state = kwargs.get('state', None) + kwargs['skip_refresh'] = kwargs.get('skip_refresh', state is not None) + super().__init__(service, path, **kwargs) + self._state = state + + @property + def clear_password(self): + return self.content.get('clear_password') + + @property + def encrypted_password(self): + return self.content.get('encr_password') + + @property + def realm(self): + return self.content.get('realm') + + @property + def username(self): + return self.content.get('username') + + +class StoragePasswords(Collection): + """This class provides access to the storage passwords from this Splunk + instance. Retrieve this collection using :meth:`Service.storage_passwords`. + """ + + def __init__(self, service): + if service.namespace.owner == '-' or service.namespace.app == '-': + raise ValueError("StoragePasswords cannot have wildcards in namespace.") + super().__init__(service, PATH_STORAGE_PASSWORDS, item=StoragePassword) + + def create(self, password, username, realm=None): + """ Creates a storage password. + + A `StoragePassword` can be identified by , or by : if the + optional realm parameter is also provided. + + :param password: The password for the credentials - this is the only part of the credentials that will be stored securely. + :type name: ``string`` + :param username: The username for the credentials. + :type name: ``string`` + :param realm: The credential realm. (optional) + :type name: ``string`` + + :return: The :class:`StoragePassword` object created. + """ + if not isinstance(username, str): + raise ValueError(f"Invalid name: {repr(username)}") + + if realm is None: + response = self.post(password=password, name=username) + else: + response = self.post(password=password, realm=realm, name=username) + + if response.status != 201: + raise ValueError(f"Unexpected status code {response.status} returned from creating a stanza") + + entries = _load_atom_entries(response) + state = _parse_atom_entry(entries[0]) + storage_password = StoragePassword(self.service, self._entity_path(state), state=state, skip_refresh=True) + + return storage_password + + def delete(self, username, realm=None): + """Delete a storage password by username and/or realm. + + The identifier can be passed in through the username parameter as + or :, but the preferred way is by + passing in the username and realm parameters. + + :param username: The username for the credentials, or : if the realm parameter is omitted. + :type name: ``string`` + :param realm: The credential realm. (optional) + :type name: ``string`` + :return: The `StoragePassword` collection. + :rtype: ``self`` + """ + if realm is None: + # This case makes the username optional, so + # the full name can be passed in as realm. + # Assume it's already encoded. + name = username + else: + # Encode each component separately + name = UrlEncoded(realm, encode_slash=True) + ":" + UrlEncoded(username, encode_slash=True) + + # Append the : expected at the end of the name + if name[-1] != ":": + name = name + ":" + return Collection.delete(self, name) + + +class AlertGroup(Entity): + """This class represents a group of fired alerts for a saved search. Access + it using the :meth:`alerts` property.""" + + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + def __len__(self): + return self.count + + @property + def alerts(self): + """Returns a collection of triggered alerts. + + :return: A :class:`Collection` of triggered alerts. + """ + return Collection(self.service, self.path) + + @property + def count(self): + """Returns the count of triggered alerts. + + :return: The triggered alert count. + :rtype: ``integer`` + """ + return int(self.content.get('triggered_alert_count', 0)) + + +class Indexes(Collection): + """This class contains the collection of indexes in this Splunk instance. + Retrieve this collection using :meth:`Service.indexes`. + """ + + def get_default(self): + """ Returns the name of the default index. + + :return: The name of the default index. + + """ + index = self['_audit'] + return index['defaultDatabase'] + + def delete(self, name): + """ Deletes a given index. + + **Note**: This method is only supported in Splunk 5.0 and later. + + :param name: The name of the index to delete. + :type name: ``string`` + """ + if self.service.splunk_version >= (5,): + Collection.delete(self, name) + else: + raise IllegalOperationException("Deleting indexes via the REST API is " + "not supported before Splunk version 5.") + + +class Index(Entity): + """This class represents an index and provides different operations, such as + cleaning the index, writing to the index, and so forth.""" + + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + def attach(self, host=None, source=None, sourcetype=None): + """Opens a stream (a writable socket) for writing events to the index. + + :param host: The host value for events written to the stream. + :type host: ``string`` + :param source: The source value for events written to the stream. + :type source: ``string`` + :param sourcetype: The sourcetype value for events written to the + stream. + :type sourcetype: ``string`` + + :return: A writable socket. + """ + args = {'index': self.name} + if host is not None: args['host'] = host + if source is not None: args['source'] = source + if sourcetype is not None: args['sourcetype'] = sourcetype + path = UrlEncoded(PATH_RECEIVERS_STREAM + "?" + parse.urlencode(args), skip_encode=True) + + cookie_header = self.service.token if self.service.token is _NoAuthenticationToken else self.service.token.replace("Splunk ", "") + cookie_or_auth_header = f"Authorization: Splunk {cookie_header}\r\n" + + # If we have cookie(s), use them instead of "Authorization: ..." + if self.service.has_cookies(): + cookie_header = _make_cookie_header(self.service.get_cookies().items()) + cookie_or_auth_header = f"Cookie: {cookie_header}\r\n" + + # Since we need to stream to the index connection, we have to keep + # the connection open and use the Splunk extension headers to note + # the input mode + sock = self.service.connect() + headers = [f"POST {str(self.service._abspath(path))} HTTP/1.1\r\n".encode('utf-8'), + f"Host: {self.service.host}:{int(self.service.port)}\r\n".encode('utf-8'), + b"Accept-Encoding: identity\r\n", + cookie_or_auth_header.encode('utf-8'), + b"X-Splunk-Input-Mode: Streaming\r\n", + b"\r\n"] + + for h in headers: + sock.write(h) + return sock + + @contextlib.contextmanager + def attached_socket(self, *args, **kwargs): + """Opens a raw socket in a ``with`` block to write data to Splunk. + + The arguments are identical to those for :meth:`attach`. The socket is + automatically closed at the end of the ``with`` block, even if an + exception is raised in the block. + + :param host: The host value for events written to the stream. + :type host: ``string`` + :param source: The source value for events written to the stream. + :type source: ``string`` + :param sourcetype: The sourcetype value for events written to the + stream. + :type sourcetype: ``string`` + + :returns: Nothing. + + **Example**:: + + import splunklib.client as client + s = client.connect(...) + index = s.indexes['some_index'] + with index.attached_socket(sourcetype='test') as sock: + sock.send('Test event\\r\\n') + + """ + try: + sock = self.attach(*args, **kwargs) + yield sock + finally: + sock.shutdown(socket.SHUT_RDWR) + sock.close() + + def clean(self, timeout=60): + """Deletes the contents of the index. + + This method blocks until the index is empty, because it needs to restore + values at the end of the operation. + + :param timeout: The time-out period for the operation, in seconds (the + default is 60). + :type timeout: ``integer`` + + :return: The :class:`Index`. + """ + self.refresh() + + tds = self['maxTotalDataSizeMB'] + ftp = self['frozenTimePeriodInSecs'] + was_disabled_initially = self.disabled + try: + if not was_disabled_initially and self.service.splunk_version < (5,): + # Need to disable the index first on Splunk 4.x, + # but it doesn't work to disable it on 5.0. + self.disable() + self.update(maxTotalDataSizeMB=1, frozenTimePeriodInSecs=1) + self.roll_hot_buckets() + + # Wait until event count goes to 0. + start = datetime.now() + diff = timedelta(seconds=timeout) + while self.content.totalEventCount != '0' and datetime.now() < start + diff: + sleep(1) + self.refresh() + + if self.content.totalEventCount != '0': + raise OperationError( + f"Cleaning index {self.name} took longer than {timeout} seconds; timing out.") + finally: + # Restore original values + self.update(maxTotalDataSizeMB=tds, frozenTimePeriodInSecs=ftp) + if not was_disabled_initially and self.service.splunk_version < (5,): + # Re-enable the index if it was originally enabled and we messed with it. + self.enable() + + return self + + def roll_hot_buckets(self): + """Performs rolling hot buckets for this index. + + :return: The :class:`Index`. + """ + self.post("roll-hot-buckets") + return self + + def submit(self, event, host=None, source=None, sourcetype=None): + """Submits a single event to the index using ``HTTP POST``. + + :param event: The event to submit. + :type event: ``string`` + :param `host`: The host value of the event. + :type host: ``string`` + :param `source`: The source value of the event. + :type source: ``string`` + :param `sourcetype`: The sourcetype value of the event. + :type sourcetype: ``string`` + + :return: The :class:`Index`. + """ + args = {'index': self.name} + if host is not None: args['host'] = host + if source is not None: args['source'] = source + if sourcetype is not None: args['sourcetype'] = sourcetype + + self.service.post(PATH_RECEIVERS_SIMPLE, body=event, **args) + return self + + # kwargs: host, host_regex, host_segment, rename-source, sourcetype + def upload(self, filename, **kwargs): + """Uploads a file for immediate indexing. + + **Note**: The file must be locally accessible from the server. + + :param filename: The name of the file to upload. The file can be a + plain, compressed, or archived file. + :type filename: ``string`` + :param kwargs: Additional arguments (optional). For more about the + available parameters, see `Index parameters `_ on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The :class:`Index`. + """ + kwargs['index'] = self.name + path = 'data/inputs/oneshot' + self.service.post(path, name=filename, **kwargs) + return self + + +class Input(Entity): + """This class represents a Splunk input. This class is the base for all + typed input classes and is also used when the client does not recognize an + input kind. + """ + + def __init__(self, service, path, kind=None, **kwargs): + # kind can be omitted (in which case it is inferred from the path) + # Otherwise, valid values are the paths from data/inputs ("udp", + # "monitor", "tcp/raw"), or two special cases: "tcp" (which is "tcp/raw") + # and "splunktcp" (which is "tcp/cooked"). + Entity.__init__(self, service, path, **kwargs) + if kind is None: + path_segments = path.split('/') + i = path_segments.index('inputs') + 1 + if path_segments[i] == 'tcp': + self.kind = path_segments[i] + '/' + path_segments[i + 1] + else: + self.kind = path_segments[i] + else: + self.kind = kind + + # Handle old input kind names. + if self.kind == 'tcp': + self.kind = 'tcp/raw' + if self.kind == 'splunktcp': + self.kind = 'tcp/cooked' + + def update(self, **kwargs): + """Updates the server with any changes you've made to the current input + along with any additional arguments you specify. + + :param kwargs: Additional arguments (optional). For more about the + available parameters, see `Input parameters `_ on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The input this method was called on. + :rtype: class:`Input` + """ + # UDP and TCP inputs require special handling due to their restrictToHost + # field. For all other inputs kinds, we can dispatch to the superclass method. + if self.kind not in ['tcp', 'splunktcp', 'tcp/raw', 'tcp/cooked', 'udp']: + return super().update(**kwargs) + else: + # The behavior of restrictToHost is inconsistent across input kinds and versions of Splunk. + # In Splunk 4.x, the name of the entity is only the port, independent of the value of + # restrictToHost. In Splunk 5.0 this changed so the name will be of the form :. + # In 5.0 and 5.0.1, if you don't supply the restrictToHost value on every update, it will + # remove the host restriction from the input. As of 5.0.2 you simply can't change restrictToHost + # on an existing input. + + # The logic to handle all these cases: + # - Throw an exception if the user tries to set restrictToHost on an existing input + # for *any* version of Splunk. + # - Set the existing restrictToHost value on the update args internally so we don't + # cause it to change in Splunk 5.0 and 5.0.1. + to_update = kwargs.copy() + + if 'restrictToHost' in kwargs: + raise IllegalOperationException("Cannot set restrictToHost on an existing input with the SDK.") + if 'restrictToHost' in self._state.content and self.kind != 'udp': + to_update['restrictToHost'] = self._state.content['restrictToHost'] + + # Do the actual update operation. + return super().update(**to_update) + + +# Inputs is a "kinded" collection, which is a heterogenous collection where +# each item is tagged with a kind, that provides a single merged view of all +# input kinds. +class Inputs(Collection): + """This class represents a collection of inputs. The collection is + heterogeneous and each member of the collection contains a *kind* property + that indicates the specific type of input. + Retrieve this collection using :meth:`Service.inputs`.""" + + def __init__(self, service, kindmap=None): + Collection.__init__(self, service, PATH_INPUTS, item=Input) + + def __getitem__(self, key): + # The key needed to retrieve the input needs it's parenthesis to be URL encoded + # based on the REST API for input + # + if isinstance(key, tuple) and len(key) == 2: + # Fetch a single kind + key, kind = key + key = UrlEncoded(key, encode_slash=True) + try: + response = self.get(self.kindpath(kind) + "/" + key) + entries = self._load_list(response) + if len(entries) > 1: + raise AmbiguousReferenceException(f"Found multiple inputs of kind {kind} named {key}.") + if len(entries) == 0: + raise KeyError((key, kind)) + return entries[0] + except HTTPError as he: + if he.status == 404: # No entity matching kind and key + raise KeyError((key, kind)) + else: + raise + else: + # Iterate over all the kinds looking for matches. + kind = None + candidate = None + key = UrlEncoded(key, encode_slash=True) + for kind in self.kinds: + try: + response = self.get(kind + "/" + key) + entries = self._load_list(response) + if len(entries) > 1: + raise AmbiguousReferenceException(f"Found multiple inputs of kind {kind} named {key}.") + if len(entries) == 0: + pass + else: + if candidate is not None: # Already found at least one candidate + raise AmbiguousReferenceException( + f"Found multiple inputs named {key}, please specify a kind") + candidate = entries[0] + except HTTPError as he: + if he.status == 404: + pass # Just carry on to the next kind. + else: + raise + if candidate is None: + raise KeyError(key) # Never found a match. + return candidate + + def __contains__(self, key): + if isinstance(key, tuple) and len(key) == 2: + # If we specify a kind, this will shortcut properly + try: + self.__getitem__(key) + return True + except KeyError: + return False + else: + # Without a kind, we want to minimize the number of round trips to the server, so we + # reimplement some of the behavior of __getitem__ in order to be able to stop searching + # on the first hit. + for kind in self.kinds: + try: + response = self.get(self.kindpath(kind) + "/" + key) + entries = self._load_list(response) + if len(entries) > 0: + return True + except HTTPError as he: + if he.status == 404: + pass # Just carry on to the next kind. + else: + raise + return False + + def create(self, name, kind, **kwargs): + """Creates an input of a specific kind in this collection, with any + arguments you specify. + + :param `name`: The input name. + :type name: ``string`` + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + :param `kwargs`: Additional arguments (optional). For more about the + available parameters, see `Input parameters `_ on Splunk Developer Portal. + + :type kwargs: ``dict`` + + :return: The new :class:`Input`. + """ + kindpath = self.kindpath(kind) + self.post(kindpath, name=name, **kwargs) + + # If we created an input with restrictToHost set, then + # its path will be :, not just , + # and we have to adjust accordingly. + + # Url encodes the name of the entity. + name = UrlEncoded(name, encode_slash=True) + path = _path( + self.path + kindpath, + f"{kwargs['restrictToHost']}:{name}" if 'restrictToHost' in kwargs else name + ) + return Input(self.service, path, kind) + + def delete(self, name, kind=None): + """Removes an input from the collection. + + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + :param name: The name of the input to remove. + :type name: ``string`` + + :return: The :class:`Inputs` collection. + """ + if kind is None: + self.service.delete(self[name].path) + else: + self.service.delete(self[name, kind].path) + return self + + def itemmeta(self, kind): + """Returns metadata for the members of a given kind. + + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + + :return: The metadata. + :rtype: class:``splunklib.data.Record`` + """ + response = self.get(f"{self._kindmap[kind]}/_new") + content = _load_atom(response, MATCH_ENTRY_CONTENT) + return _parse_atom_metadata(content) + + def _get_kind_list(self, subpath=None): + if subpath is None: + subpath = [] + + kinds = [] + response = self.get('/'.join(subpath)) + content = _load_atom_entries(response) + for entry in content: + this_subpath = subpath + [entry.title] + # The "all" endpoint doesn't work yet. + # The "tcp/ssl" endpoint is not a real input collection. + if entry.title == 'all' or this_subpath == ['tcp', 'ssl']: + continue + if 'create' in [x.rel for x in entry.link]: + path = '/'.join(subpath + [entry.title]) + kinds.append(path) + else: + subkinds = self._get_kind_list(subpath + [entry.title]) + kinds.extend(subkinds) + return kinds + + @property + def kinds(self): + """Returns the input kinds on this Splunk instance. + + :return: The list of input kinds. + :rtype: ``list`` + """ + return self._get_kind_list() + + def kindpath(self, kind): + """Returns a path to the resources for a given input kind. + + :param `kind`: The kind of input: + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kind: ``string`` + + :return: The relative endpoint path. + :rtype: ``string`` + """ + if kind == 'tcp': + return UrlEncoded('tcp/raw', skip_encode=True) + if kind == 'splunktcp': + return UrlEncoded('tcp/cooked', skip_encode=True) + return UrlEncoded(kind, skip_encode=True) + + def list(self, *kinds, **kwargs): + """Returns a list of inputs that are in the :class:`Inputs` collection. + You can also filter by one or more input kinds. + + This function iterates over all possible inputs, regardless of any arguments you + specify. Because the :class:`Inputs` collection is the union of all the inputs of each + kind, this method implements parameters such as "count", "search", and so + on at the Python level once all the data has been fetched. The exception + is when you specify a single input kind, and then this method makes a single request + with the usual semantics for parameters. + + :param kinds: The input kinds to return (optional). + + - "ad": Active Directory + + - "monitor": Files and directories + + - "registry": Windows Registry + + - "script": Scripts + + - "splunktcp": TCP, processed + + - "tcp": TCP, unprocessed + + - "udp": UDP + + - "win-event-log-collections": Windows event log + + - "win-perfmon": Performance monitoring + + - "win-wmi-collections": WMI + + :type kinds: ``string`` + :param kwargs: Additional arguments (optional): + + - "count" (``integer``): The maximum number of items to return. + + - "offset" (``integer``): The offset of the first item to return. + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + + :return: A list of input kinds. + :rtype: ``list`` + """ + if len(kinds) == 0: + kinds = self.kinds + if len(kinds) == 1: + kind = kinds[0] + logger.debug("Inputs.list taking short circuit branch for single kind.") + path = self.kindpath(kind) + logger.debug("Path for inputs: %s", path) + try: + path = UrlEncoded(path, skip_encode=True) + response = self.get(path, **kwargs) + except HTTPError as he: + if he.status == 404: # No inputs of this kind + return [] + entities = [] + entries = _load_atom_entries(response) + if entries is None: + return [] # No inputs in a collection comes back with no feed or entry in the XML + for entry in entries: + state = _parse_atom_entry(entry) + # Unquote the URL, since all URL encoded in the SDK + # should be of type UrlEncoded, and all str should not + # be URL encoded. + path = parse.unquote(state.links.alternate) + entity = Input(self.service, path, kind, state=state) + entities.append(entity) + return entities + + search = kwargs.get('search', '*') + + entities = [] + for kind in kinds: + response = None + try: + kind = UrlEncoded(kind, skip_encode=True) + response = self.get(self.kindpath(kind), search=search) + except HTTPError as e: + if e.status == 404: + continue # No inputs of this kind + else: + raise + + entries = _load_atom_entries(response) + if entries is None: continue # No inputs to process + for entry in entries: + state = _parse_atom_entry(entry) + # Unquote the URL, since all URL encoded in the SDK + # should be of type UrlEncoded, and all str should not + # be URL encoded. + path = parse.unquote(state.links.alternate) + entity = Input(self.service, path, kind, state=state) + entities.append(entity) + if 'offset' in kwargs: + entities = entities[kwargs['offset']:] + if 'count' in kwargs: + entities = entities[:kwargs['count']] + if kwargs.get('sort_mode', None) == 'alpha': + sort_field = kwargs.get('sort_field', 'name') + if sort_field == 'name': + f = lambda x: x.name.lower() + else: + f = lambda x: x[sort_field].lower() + entities = sorted(entities, key=f) + if kwargs.get('sort_mode', None) == 'alpha_case': + sort_field = kwargs.get('sort_field', 'name') + if sort_field == 'name': + f = lambda x: x.name + else: + f = lambda x: x[sort_field] + entities = sorted(entities, key=f) + if kwargs.get('sort_dir', 'asc') == 'desc': + entities = list(reversed(entities)) + return entities + + def __iter__(self, **kwargs): + for item in self.iter(**kwargs): + yield item + + def iter(self, **kwargs): + """ Iterates over the collection of inputs. + + :param kwargs: Additional arguments (optional): + + - "count" (``integer``): The maximum number of items to return. + + - "offset" (``integer``): The offset of the first item to return. + + - "search" (``string``): The search query to filter responses. + + - "sort_dir" (``string``): The direction to sort returned items: + "asc" or "desc". + + - "sort_key" (``string``): The field to use for sorting (optional). + + - "sort_mode" (``string``): The collating sequence for sorting + returned items: "auto", "alpha", "alpha_case", or "num". + + :type kwargs: ``dict`` + """ + for item in self.list(**kwargs): + yield item + + def oneshot(self, path, **kwargs): + """ Creates a oneshot data input, which is an upload of a single file + for one-time indexing. + + :param path: The path and filename. + :type path: ``string`` + :param kwargs: Additional arguments (optional). For more about the + available parameters, see `Input parameters `_ on Splunk Developer Portal. + :type kwargs: ``dict`` + """ + self.post('oneshot', name=path, **kwargs) + + +class Job(Entity): + """This class represents a search job.""" + + def __init__(self, service, sid, **kwargs): + # Default to v2 in Splunk Version 9+ + path = "{path}{sid}" + # Formatting path based on the Splunk Version + if service.disable_v2_api: + path = path.format(path=PATH_JOBS, sid=sid) + else: + path = path.format(path=PATH_JOBS_V2, sid=sid) + + Entity.__init__(self, service, path, skip_refresh=True, **kwargs) + self.sid = sid + + # The Job entry record is returned at the root of the response + def _load_atom_entry(self, response): + return _load_atom(response).entry + + def cancel(self): + """Stops the current search and deletes the results cache. + + :return: The :class:`Job`. + """ + try: + self.post("control", action="cancel") + except HTTPError as he: + if he.status == 404: + # The job has already been cancelled, so + # cancelling it twice is a nop. + pass + else: + raise + return self + + def disable_preview(self): + """Disables preview for this job. + + :return: The :class:`Job`. + """ + self.post("control", action="disablepreview") + return self + + def enable_preview(self): + """Enables preview for this job. + + **Note**: Enabling preview might slow search considerably. + + :return: The :class:`Job`. + """ + self.post("control", action="enablepreview") + return self + + def events(self, **kwargs): + """Returns a streaming handle to this job's events. + + :param kwargs: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/events + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's events. + """ + kwargs['segmentation'] = kwargs.get('segmentation', 'none') + + # Search API v1(GET) and v2(POST) + if self.service.disable_v2_api: + return self.get("events", **kwargs).body + return self.post("events", **kwargs).body + + def finalize(self): + """Stops the job and provides intermediate results for retrieval. + + :return: The :class:`Job`. + """ + self.post("control", action="finalize") + return self + + def is_done(self): + """Indicates whether this job finished running. + + :return: ``True`` if the job is done, ``False`` if not. + :rtype: ``boolean`` + """ + if not self.is_ready(): + return False + done = (self._state.content['isDone'] == '1') + return done + + def is_ready(self): + """Indicates whether this job is ready for querying. + + :return: ``True`` if the job is ready, ``False`` if not. + :rtype: ``boolean`` + + """ + response = self.get() + if response.status == 204: + return False + self._state = self.read(response) + ready = self._state.content['dispatchState'] not in ['QUEUED', 'PARSING'] + return ready + + @property + def name(self): + """Returns the name of the search job, which is the search ID (SID). + + :return: The search ID. + :rtype: ``string`` + """ + return self.sid + + def pause(self): + """Suspends the current search. + + :return: The :class:`Job`. + """ + self.post("control", action="pause") + return self + + def results(self, **query_params): + """Returns a streaming handle to this job's search results. To get a nice, Pythonic iterator, pass the handle + to :class:`splunklib.results.JSONResultsReader` along with the query param "output_mode='json'", as in:: + + import splunklib.client as client + import splunklib.results as results + from time import sleep + service = client.connect(...) + job = service.jobs.create("search * | head 5") + while not job.is_done(): + sleep(.2) + rr = results.JSONResultsReader(job.results(output_mode='json')) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print(f'{result.type}: {result.message}') + elif isinstance(result, dict): + # Normal events are returned as dicts + print(result) + assert rr.is_preview == False + + Results are not available until the job has finished. If called on + an unfinished job, the result is an empty event set. + + This method makes a single roundtrip + to the server, plus at most two additional round trips if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param query_params: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/results + `_. + :type query_params: ``dict`` + + :return: The ``InputStream`` IO handle to this job's results. + """ + query_params['segmentation'] = query_params.get('segmentation', 'none') + + # Search API v1(GET) and v2(POST) + if self.service.disable_v2_api: + return self.get("results", **query_params).body + return self.post("results", **query_params).body + + def preview(self, **query_params): + """Returns a streaming handle to this job's preview search results. + + Unlike :class:`splunklib.results.JSONResultsReader`along with the query param "output_mode='json'", + which requires a job to be finished to return any results, the ``preview`` method returns any results that + have been generated so far, whether the job is running or not. The returned search results are the raw data + from the server. Pass the handle returned to :class:`splunklib.results.JSONResultsReader` to get a nice, + Pythonic iterator over objects, as in:: + + import splunklib.client as client + import splunklib.results as results + service = client.connect(...) + job = service.jobs.create("search * | head 5") + rr = results.JSONResultsReader(job.preview(output_mode='json')) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print(f'{result.type}: {result.message}') + elif isinstance(result, dict): + # Normal events are returned as dicts + print(result) + if rr.is_preview: + print("Preview of a running search job.") + else: + print("Job is finished. Results are final.") + + This method makes one roundtrip to the server, plus at most + two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param query_params: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/results_preview + `_ + in the REST API documentation. + :type query_params: ``dict`` + + :return: The ``InputStream`` IO handle to this job's preview results. + """ + query_params['segmentation'] = query_params.get('segmentation', 'none') + + # Search API v1(GET) and v2(POST) + if self.service.disable_v2_api: + return self.get("results_preview", **query_params).body + return self.post("results_preview", **query_params).body + + def searchlog(self, **kwargs): + """Returns a streaming handle to this job's search log. + + :param `kwargs`: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/search.log + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's search log. + """ + return self.get("search.log", **kwargs).body + + def set_priority(self, value): + """Sets this job's search priority in the range of 0-10. + + Higher numbers indicate higher priority. Unless splunkd is + running as *root*, you can only decrease the priority of a running job. + + :param `value`: The search priority. + :type value: ``integer`` + + :return: The :class:`Job`. + """ + self.post('control', action="setpriority", priority=value) + return self + + def summary(self, **kwargs): + """Returns a streaming handle to this job's summary. + + :param `kwargs`: Additional parameters (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/summary + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's summary. + """ + return self.get("summary", **kwargs).body + + def timeline(self, **kwargs): + """Returns a streaming handle to this job's timeline results. + + :param `kwargs`: Additional timeline arguments (optional). For a list of valid + parameters, see `GET search/jobs/{search_id}/timeline + `_ + in the REST API documentation. + :type kwargs: ``dict`` + + :return: The ``InputStream`` IO handle to this job's timeline. + """ + return self.get("timeline", **kwargs).body + + def touch(self): + """Extends the expiration time of the search to the current time (now) plus + the time-to-live (ttl) value. + + :return: The :class:`Job`. + """ + self.post("control", action="touch") + return self + + def set_ttl(self, value): + """Set the job's time-to-live (ttl) value, which is the time before the + search job expires and is still available. + + :param `value`: The ttl value, in seconds. + :type value: ``integer`` + + :return: The :class:`Job`. + """ + self.post("control", action="setttl", ttl=value) + return self + + def unpause(self): + """Resumes the current search, if paused. + + :return: The :class:`Job`. + """ + self.post("control", action="unpause") + return self + + +class Jobs(Collection): + """This class represents a collection of search jobs. Retrieve this + collection using :meth:`Service.jobs`.""" + + def __init__(self, service): + # Splunk 9 introduces the v2 endpoint + if not service.disable_v2_api: + path = PATH_JOBS_V2 + else: + path = PATH_JOBS + Collection.__init__(self, service, path, item=Job) + # The count value to say list all the contents of this + # Collection is 0, not -1 as it is on most. + self.null_count = 0 + + def _load_list(self, response): + # Overridden because Job takes a sid instead of a path. + entries = _load_atom_entries(response) + if entries is None: return [] + entities = [] + for entry in entries: + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + entry['content']['sid'], + state=state) + entities.append(entity) + return entities + + def create(self, query, **kwargs): + """ Creates a search using a search query and any additional parameters + you provide. + + :param query: The search query. + :type query: ``string`` + :param kwargs: Additiona parameters (optional). For a list of available + parameters, see `Search job parameters + `_ + on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The :class:`Job`. + """ + if kwargs.get("exec_mode", None) == "oneshot": + raise TypeError("Cannot specify exec_mode=oneshot; use the oneshot method instead.") + response = self.post(search=query, **kwargs) + sid = _load_sid(response, kwargs.get("output_mode", None)) + return Job(self.service, sid) + + def export(self, query, **params): + """Runs a search and immediately starts streaming preview events. This method returns a streaming handle to + this job's events as an XML document from the server. To parse this stream into usable Python objects, + pass the handle to :class:`splunklib.results.JSONResultsReader` along with the query param + "output_mode='json'":: + + import splunklib.client as client + import splunklib.results as results + service = client.connect(...) + rr = results.JSONResultsReader(service.jobs.export("search * | head 5",output_mode='json')) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print(f'{result.type}: {result.message}') + elif isinstance(result, dict): + # Normal events are returned as dicts + print(result) + assert rr.is_preview == False + + Running an export search is more efficient as it streams the results + directly to you, rather than having to write them out to disk and make + them available later. As soon as results are ready, you will receive + them. + + The ``export`` method makes a single roundtrip to the server (as opposed + to two for :meth:`create` followed by :meth:`preview`), plus at most two + more if the ``autologin`` field of :func:`connect` is set to ``True``. + + :raises `ValueError`: Raised for invalid queries. + :param query: The search query. + :type query: ``string`` + :param params: Additional arguments (optional). For a list of valid + parameters, see `GET search/jobs/export + `_ + in the REST API documentation. + :type params: ``dict`` + + :return: The ``InputStream`` IO handle to raw XML returned from the server. + """ + if "exec_mode" in params: + raise TypeError("Cannot specify an exec_mode to export.") + params['segmentation'] = params.get('segmentation', 'none') + return self.post(path_segment="export", + search=query, + **params).body + + def itemmeta(self): + """There is no metadata available for class:``Jobs``. + + Any call to this method raises a class:``NotSupportedError``. + + :raises: class:``NotSupportedError`` + """ + raise NotSupportedError() + + def oneshot(self, query, **params): + """Run a oneshot search and returns a streaming handle to the results. + + The ``InputStream`` object streams fragments from the server. To parse this stream into usable Python + objects, pass the handle to :class:`splunklib.results.JSONResultsReader` along with the query param + "output_mode='json'" :: + + import splunklib.client as client + import splunklib.results as results + service = client.connect(...) + rr = results.JSONResultsReader(service.jobs.oneshot("search * | head 5",output_mode='json')) + for result in rr: + if isinstance(result, results.Message): + # Diagnostic messages may be returned in the results + print(f'{result.type}: {result.message}') + elif isinstance(result, dict): + # Normal events are returned as dicts + print(result) + assert rr.is_preview == False + + The ``oneshot`` method makes a single roundtrip to the server (as opposed + to two for :meth:`create` followed by :meth:`results`), plus at most two more + if the ``autologin`` field of :func:`connect` is set to ``True``. + + :raises ValueError: Raised for invalid queries. + + :param query: The search query. + :type query: ``string`` + :param params: Additional arguments (optional): + + - "output_mode": Specifies the output format of the results (XML, + JSON, or CSV). + + - "earliest_time": Specifies the earliest time in the time range to + search. The time string can be a UTC time (with fractional seconds), + a relative time specifier (to now), or a formatted time string. + + - "latest_time": Specifies the latest time in the time range to + search. The time string can be a UTC time (with fractional seconds), + a relative time specifier (to now), or a formatted time string. + + - "rf": Specifies one or more fields to add to the search. + + :type params: ``dict`` + + :return: The ``InputStream`` IO handle to raw XML returned from the server. + """ + if "exec_mode" in params: + raise TypeError("Cannot specify an exec_mode to oneshot.") + params['segmentation'] = params.get('segmentation', 'none') + return self.post(search=query, + exec_mode="oneshot", + **params).body + + +class Loggers(Collection): + """This class represents a collection of service logging categories. + Retrieve this collection using :meth:`Service.loggers`.""" + + def __init__(self, service): + Collection.__init__(self, service, PATH_LOGGER) + + def itemmeta(self): + """There is no metadata available for class:``Loggers``. + + Any call to this method raises a class:``NotSupportedError``. + + :raises: class:``NotSupportedError`` + """ + raise NotSupportedError() + + +class Message(Entity): + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + @property + def value(self): + """Returns the message value. + + :return: The message value. + :rtype: ``string`` + """ + return self[self.name] + + +class ModularInputKind(Entity): + """This class contains the different types of modular inputs. Retrieve this + collection using :meth:`Service.modular_input_kinds`. + """ + + def __contains__(self, name): + args = self.state.content['endpoints']['args'] + if name in args: + return True + return Entity.__contains__(self, name) + + def __getitem__(self, name): + args = self.state.content['endpoint']['args'] + if name in args: + return args['item'] + return Entity.__getitem__(self, name) + + @property + def arguments(self): + """A dictionary of all the arguments supported by this modular input kind. + + The keys in the dictionary are the names of the arguments. The values are + another dictionary giving the metadata about that argument. The possible + keys in that dictionary are ``"title"``, ``"description"``, ``"required_on_create``", + ``"required_on_edit"``, ``"data_type"``. Each value is a string. It should be one + of ``"true"`` or ``"false"`` for ``"required_on_create"`` and ``"required_on_edit"``, + and one of ``"boolean"``, ``"string"``, or ``"number``" for ``"data_type"``. + + :return: A dictionary describing the arguments this modular input kind takes. + :rtype: ``dict`` + """ + return self.state.content['endpoint']['args'] + + def update(self, **kwargs): + """Raises an error. Modular input kinds are read only.""" + raise IllegalOperationException("Modular input kinds cannot be updated via the REST API.") + + +class SavedSearch(Entity): + """This class represents a saved search.""" + + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + def acknowledge(self): + """Acknowledges the suppression of alerts from this saved search and + resumes alerting. + + :return: The :class:`SavedSearch`. + """ + self.post("acknowledge") + return self + + @property + def alert_count(self): + """Returns the number of alerts fired by this saved search. + + :return: The number of alerts fired by this saved search. + :rtype: ``integer`` + """ + return int(self._state.content.get('triggered_alert_count', 0)) + + def dispatch(self, **kwargs): + """Runs the saved search and returns the resulting search job. + + :param `kwargs`: Additional dispatch arguments (optional). For details, + see the `POST saved/searches/{name}/dispatch + `_ + endpoint in the REST API documentation. + :type kwargs: ``dict`` + :return: The :class:`Job`. + """ + response = self.post("dispatch", **kwargs) + sid = _load_sid(response, kwargs.get("output_mode", None)) + return Job(self.service, sid) + + @property + def fired_alerts(self): + """Returns the collection of fired alerts (a fired alert group) + corresponding to this saved search's alerts. + + :raises IllegalOperationException: Raised when the search is not scheduled. + + :return: A collection of fired alerts. + :rtype: :class:`AlertGroup` + """ + if self['is_scheduled'] == '0': + raise IllegalOperationException('Unscheduled saved searches have no alerts.') + c = Collection( + self.service, + self.service._abspath(PATH_FIRED_ALERTS + self.name, + owner=self._state.access.owner, + app=self._state.access.app, + sharing=self._state.access.sharing), + item=AlertGroup) + return c + + def history(self, **kwargs): + """Returns a list of search jobs corresponding to this saved search. + + :param `kwargs`: Additional arguments (optional). + :type kwargs: ``dict`` + + :return: A list of :class:`Job` objects. + """ + response = self.get("history", **kwargs) + entries = _load_atom_entries(response) + if entries is None: return [] + jobs = [] + for entry in entries: + job = Job(self.service, entry.title) + jobs.append(job) + return jobs + + def update(self, search=None, **kwargs): + """Updates the server with any changes you've made to the current saved + search along with any additional arguments you specify. + + :param `search`: The search query (optional). + :type search: ``string`` + :param `kwargs`: Additional arguments (optional). For a list of available + parameters, see `Saved search parameters + `_ + on Splunk Developer Portal. + :type kwargs: ``dict`` + + :return: The :class:`SavedSearch`. + """ + # Updates to a saved search *require* that the search string be + # passed, so we pass the current search string if a value wasn't + # provided by the caller. + if search is None: search = self.content.search + Entity.update(self, search=search, **kwargs) + return self + + def scheduled_times(self, earliest_time='now', latest_time='+1h'): + """Returns the times when this search is scheduled to run. + + By default this method returns the times in the next hour. For different + time ranges, set *earliest_time* and *latest_time*. For example, + for all times in the last day use "earliest_time=-1d" and + "latest_time=now". + + :param earliest_time: The earliest time. + :type earliest_time: ``string`` + :param latest_time: The latest time. + :type latest_time: ``string`` + + :return: The list of search times. + """ + response = self.get("scheduled_times", + earliest_time=earliest_time, + latest_time=latest_time) + data = self._load_atom_entry(response) + rec = _parse_atom_entry(data) + times = [datetime.fromtimestamp(int(t)) + for t in rec.content.scheduled_times] + return times + + def suppress(self, expiration): + """Skips any scheduled runs of this search in the next *expiration* + number of seconds. + + :param expiration: The expiration period, in seconds. + :type expiration: ``integer`` + + :return: The :class:`SavedSearch`. + """ + self.post("suppress", expiration=expiration) + return self + + @property + def suppressed(self): + """Returns the number of seconds that this search is blocked from running + (possibly 0). + + :return: The number of seconds. + :rtype: ``integer`` + """ + r = self._run_action("suppress") + if r.suppressed == "1": + return int(r.expiration) + return 0 + + def unsuppress(self): + """Cancels suppression and makes this search run as scheduled. + + :return: The :class:`SavedSearch`. + """ + self.post("suppress", expiration="0") + return self + + +class SavedSearches(Collection): + """This class represents a collection of saved searches. Retrieve this + collection using :meth:`Service.saved_searches`.""" + + def __init__(self, service): + Collection.__init__( + self, service, PATH_SAVED_SEARCHES, item=SavedSearch) + + def create(self, name, search, **kwargs): + """ Creates a saved search. + + :param name: The name for the saved search. + :type name: ``string`` + :param search: The search query. + :type search: ``string`` + :param kwargs: Additional arguments (optional). For a list of available + parameters, see `Saved search parameters + `_ + on Splunk Developer Portal. + :type kwargs: ``dict`` + :return: The :class:`SavedSearches` collection. + """ + return Collection.create(self, name, search=search, **kwargs) + + +class Settings(Entity): + """This class represents configuration settings for a Splunk service. + Retrieve this collection using :meth:`Service.settings`.""" + + def __init__(self, service, **kwargs): + Entity.__init__(self, service, "/services/server/settings", **kwargs) + + # Updates on the settings endpoint are POSTed to server/settings/settings. + def update(self, **kwargs): + """Updates the settings on the server using the arguments you provide. + + :param kwargs: Additional arguments. For a list of valid arguments, see + `POST server/settings/{name} + `_ + in the REST API documentation. + :type kwargs: ``dict`` + :return: The :class:`Settings` collection. + """ + self.service.post("/services/server/settings/settings", **kwargs) + return self + + +class User(Entity): + """This class represents a Splunk user. + """ + + @property + def role_entities(self): + """Returns a list of roles assigned to this user. + + :return: The list of roles. + :rtype: ``list`` + """ + all_role_names = [r.name for r in self.service.roles.list()] + return [self.service.roles[name] for name in self.content.roles if name in all_role_names] + + +# Splunk automatically lowercases new user names so we need to match that +# behavior here to ensure that the subsequent member lookup works correctly. +class Users(Collection): + """This class represents the collection of Splunk users for this instance of + Splunk. Retrieve this collection using :meth:`Service.users`. + """ + + def __init__(self, service): + Collection.__init__(self, service, PATH_USERS, item=User) + + def __getitem__(self, key): + return Collection.__getitem__(self, key.lower()) + + def __contains__(self, name): + return Collection.__contains__(self, name.lower()) + + def create(self, username, password, roles, **params): + """Creates a new user. + + This function makes two roundtrips to the server, plus at most + two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param username: The username. + :type username: ``string`` + :param password: The password. + :type password: ``string`` + :param roles: A single role or list of roles for the user. + :type roles: ``string`` or ``list`` + :param params: Additional arguments (optional). For a list of available + parameters, see `User authentication parameters + `_ + on Splunk Developer Portal. + :type params: ``dict`` + + :return: The new user. + :rtype: :class:`User` + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + users = c.users + boris = users.create("boris", "securepassword", roles="user") + hilda = users.create("hilda", "anotherpassword", roles=["user","power"]) + """ + if not isinstance(username, str): + raise ValueError(f"Invalid username: {str(username)}") + username = username.lower() + self.post(name=username, password=password, roles=roles, **params) + # splunkd doesn't return the user in the POST response body, + # so we have to make a second round trip to fetch it. + response = self.get(username) + entry = _load_atom(response, XNAME_ENTRY).entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + parse.unquote(state.links.alternate), + state=state) + return entity + + def delete(self, name): + """ Deletes the user and returns the resulting collection of users. + + :param name: The name of the user to delete. + :type name: ``string`` + + :return: + :rtype: :class:`Users` + """ + return Collection.delete(self, name.lower()) + + +class Role(Entity): + """This class represents a user role. + """ + + def grant(self, *capabilities_to_grant): + """Grants additional capabilities to this role. + + :param capabilities_to_grant: Zero or more capabilities to grant this + role. For a list of capabilities, see + `Capabilities `_ + on Splunk Developer Portal. + :type capabilities_to_grant: ``string`` or ``list`` + :return: The :class:`Role`. + + **Example**:: + + service = client.connect(...) + role = service.roles['somerole'] + role.grant('change_own_password', 'search') + """ + possible_capabilities = self.service.capabilities + for capability in capabilities_to_grant: + if capability not in possible_capabilities: + raise NoSuchCapability(capability) + new_capabilities = self['capabilities'] + list(capabilities_to_grant) + self.post(capabilities=new_capabilities) + return self + + def revoke(self, *capabilities_to_revoke): + """Revokes zero or more capabilities from this role. + + :param capabilities_to_revoke: Zero or more capabilities to grant this + role. For a list of capabilities, see + `Capabilities `_ + on Splunk Developer Portal. + :type capabilities_to_revoke: ``string`` or ``list`` + + :return: The :class:`Role`. + + **Example**:: + + service = client.connect(...) + role = service.roles['somerole'] + role.revoke('change_own_password', 'search') + """ + possible_capabilities = self.service.capabilities + for capability in capabilities_to_revoke: + if capability not in possible_capabilities: + raise NoSuchCapability(capability) + old_capabilities = self['capabilities'] + new_capabilities = [] + for c in old_capabilities: + if c not in capabilities_to_revoke: + new_capabilities.append(c) + if not new_capabilities: + new_capabilities = '' # Empty lists don't get passed in the body, so we have to force an empty argument. + self.post(capabilities=new_capabilities) + return self + + +class Roles(Collection): + """This class represents the collection of roles in the Splunk instance. + Retrieve this collection using :meth:`Service.roles`.""" + + def __init__(self, service): + Collection.__init__(self, service, PATH_ROLES, item=Role) + + def __getitem__(self, key): + return Collection.__getitem__(self, key.lower()) + + def __contains__(self, name): + return Collection.__contains__(self, name.lower()) + + def create(self, name, **params): + """Creates a new role. + + This function makes two roundtrips to the server, plus at most + two more if + the ``autologin`` field of :func:`connect` is set to ``True``. + + :param name: Name for the role. + :type name: ``string`` + :param params: Additional arguments (optional). For a list of available + parameters, see `Roles parameters + `_ + on Splunk Developer Portal. + :type params: ``dict`` + + :return: The new role. + :rtype: :class:`Role` + + **Example**:: + + import splunklib.client as client + c = client.connect(...) + roles = c.roles + paltry = roles.create("paltry", imported_roles="user", defaultApp="search") + """ + if not isinstance(name, str): + raise ValueError(f"Invalid role name: {str(name)}") + name = name.lower() + self.post(name=name, **params) + # splunkd doesn't return the user in the POST response body, + # so we have to make a second round trip to fetch it. + response = self.get(name) + entry = _load_atom(response, XNAME_ENTRY).entry + state = _parse_atom_entry(entry) + entity = self.item( + self.service, + parse.unquote(state.links.alternate), + state=state) + return entity + + def delete(self, name): + """ Deletes the role and returns the resulting collection of roles. + + :param name: The name of the role to delete. + :type name: ``string`` + + :rtype: The :class:`Roles` + """ + return Collection.delete(self, name.lower()) + + +class Application(Entity): + """Represents a locally-installed Splunk app.""" + + @property + def setupInfo(self): + """Returns the setup information for the app. + + :return: The setup information. + """ + return self.content.get('eai:setup', None) + + def package(self): + """ Creates a compressed package of the app for archiving.""" + return self._run_action("package") + + def updateInfo(self): + """Returns any update information that is available for the app.""" + return self._run_action("update") + + +class KVStoreCollections(Collection): + def __init__(self, service): + Collection.__init__(self, service, 'storage/collections/config', item=KVStoreCollection) + + def __getitem__(self, item): + res = Collection.__getitem__(self, item) + for k, v in res.content.items(): + if "accelerated_fields" in k: + res.content[k] = json.loads(v) + return res + + def create(self, name, accelerated_fields={}, fields={}, **kwargs): + """Creates a KV Store Collection. + + :param name: name of collection to create + :type name: ``string`` + :param accelerated_fields: dictionary of accelerated_fields definitions + :type accelerated_fields: ``dict`` + :param fields: dictionary of field definitions + :type fields: ``dict`` + :param kwargs: a dictionary of additional parameters specifying indexes and field definitions + :type kwargs: ``dict`` + + :return: Result of POST request + """ + for k, v in accelerated_fields.items(): + if isinstance(v, dict): + v = json.dumps(v) + kwargs['accelerated_fields.' + k] = v + for k, v in fields.items(): + kwargs['field.' + k] = v + return self.post(name=name, **kwargs) + + +class KVStoreCollection(Entity): + @property + def data(self): + """Returns data object for this Collection. + + :rtype: :class:`KVStoreCollectionData` + """ + return KVStoreCollectionData(self) + + def update_accelerated_field(self, name, value): + """Changes the definition of a KV Store accelerated_field. + + :param name: name of accelerated_fields to change + :type name: ``string`` + :param value: new accelerated_fields definition + :type value: ``dict`` + + :return: Result of POST request + """ + kwargs = {} + kwargs['accelerated_fields.' + name] = json.dumps(value) if isinstance(value, dict) else value + return self.post(**kwargs) + + def update_field(self, name, value): + """Changes the definition of a KV Store field. + + :param name: name of field to change + :type name: ``string`` + :param value: new field definition + :type value: ``string`` + + :return: Result of POST request + """ + kwargs = {} + kwargs['field.' + name] = value + return self.post(**kwargs) + + +class KVStoreCollectionData: + """This class represents the data endpoint for a KVStoreCollection. + + Retrieve using :meth:`KVStoreCollection.data` + """ + JSON_HEADER = [('Content-Type', 'application/json')] + + def __init__(self, collection): + self.service = collection.service + self.collection = collection + self.owner, self.app, self.sharing = collection._proper_namespace() + self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name, encode_slash=True) + '/' + + def _get(self, url, **kwargs): + return self.service.get(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) + + def _post(self, url, **kwargs): + return self.service.post(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) + + def _delete(self, url, **kwargs): + return self.service.delete(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) + + def query(self, **query): + """ + Gets the results of query, with optional parameters sort, limit, skip, and fields. + + :param query: Optional parameters. Valid options are sort, limit, skip, and fields + :type query: ``dict`` + + :return: Array of documents retrieved by query. + :rtype: ``array`` + """ + + for key, value in query.items(): + if isinstance(query[key], dict): + query[key] = json.dumps(value) + + return json.loads(self._get('', **query).body.read().decode('utf-8')) + + def query_by_id(self, id): + """ + Returns object with _id = id. + + :param id: Value for ID. If not a string will be coerced to string. + :type id: ``string`` + + :return: Document with id + :rtype: ``dict`` + """ + return json.loads(self._get(UrlEncoded(str(id), encode_slash=True)).body.read().decode('utf-8')) + + def insert(self, data): + """ + Inserts item into this collection. An _id field will be generated if not assigned in the data. + + :param data: Document to insert + :type data: ``string`` + + :return: _id of inserted object + :rtype: ``dict`` + """ + if isinstance(data, dict): + data = json.dumps(data) + return json.loads( + self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + + def delete(self, query=None): + """ + Deletes all data in collection if query is absent. Otherwise, deletes all data matched by query. + + :param query: Query to select documents to delete + :type query: ``string`` + + :return: Result of DELETE request + """ + return self._delete('', **({'query': query}) if query else {}) + + def delete_by_id(self, id): + """ + Deletes document that has _id = id. + + :param id: id of document to delete + :type id: ``string`` + + :return: Result of DELETE request + """ + return self._delete(UrlEncoded(str(id), encode_slash=True)) + + def update(self, id, data): + """ + Replaces document with _id = id with data. + + :param id: _id of document to update + :type id: ``string`` + :param data: the new document to insert + :type data: ``string`` + + :return: id of replaced document + :rtype: ``dict`` + """ + if isinstance(data, dict): + data = json.dumps(data) + return json.loads(self._post(UrlEncoded(str(id), encode_slash=True), headers=KVStoreCollectionData.JSON_HEADER, + body=data).body.read().decode('utf-8')) + + def batch_find(self, *dbqueries): + """ + Returns array of results from queries dbqueries. + + :param dbqueries: Array of individual queries as dictionaries + :type dbqueries: ``array`` of ``dict`` + + :return: Results of each query + :rtype: ``array`` of ``array`` + """ + if len(dbqueries) < 1: + raise Exception('Must have at least one query.') + + data = json.dumps(dbqueries) + + return json.loads( + self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + + def batch_save(self, *documents): + """ + Inserts or updates every document specified in documents. + + :param documents: Array of documents to save as dictionaries + :type documents: ``array`` of ``dict`` + + :return: Results of update operation as overall stats + :rtype: ``dict`` + """ + if len(documents) < 1: + raise Exception('Must have at least one document.') + + data = json.dumps(documents) + + return json.loads( + self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index 16999a2a..7e8f771e 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -1,1143 +1,1143 @@ -# coding=utf-8 -# -# Copyright © 2011-2024 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Absolute imports - -import csv -import io -import os -import re -import sys -import tempfile -import traceback -from collections import namedtuple, OrderedDict -from copy import deepcopy -from io import StringIO -from itertools import chain, islice -from logging import _nameToLevel as _levelNames, getLevelName, getLogger -from shutil import make_archive -from time import time -from urllib.parse import unquote -from urllib.parse import urlsplit -from warnings import warn -from xml.etree import ElementTree -from splunklib.utils import ensure_str - - -# Relative imports -import splunklib -from . import Boolean, Option, environment -from .internals import ( - CommandLineParser, - CsvDialect, - InputHeader, - Message, - MetadataDecoder, - MetadataEncoder, - ObjectView, - Recorder, - RecordWriterV1, - RecordWriterV2, - json_encode_string) -from ..client import Service - - -# ---------------------------------------------------------------------------------------------------------------------- - -# P1 [ ] TODO: Log these issues against ChunkedExternProcessor -# -# 1. Implement requires_preop configuration setting. -# This configuration setting is currently rejected by ChunkedExternProcessor. -# -# 2. Rename type=events as type=eventing for symmetry with type=reporting and type=streaming -# Eventing commands process records on the events pipeline. This change effects ChunkedExternProcessor.cpp, -# eventing_command.py, and generating_command.py. -# -# 3. For consistency with SCPV1, commands.conf should not require filename setting when chunked = true -# The SCPV1 processor uses .py as the default filename. The ChunkedExternProcessor should do the same. - -# P1 [ ] TODO: Verify that ChunkedExternProcessor complains if a streaming_preop has a type other than 'streaming' -# It once looked like sending type='reporting' for the streaming_preop was accepted. - -# ---------------------------------------------------------------------------------------------------------------------- - -# P2 [ ] TODO: Consider bumping None formatting up to Option.Item.__str__ - - -class SearchCommand: - """ Represents a custom search command. - - """ - - def __init__(self): - - # Variables that may be used, but not altered by derived classes - - class_name = self.__class__.__name__ - - self._logger, self._logging_configuration = getLogger(class_name), environment.logging_configuration - - # Variables backing option/property values - - self._configuration = self.ConfigurationSettings(self) - self._input_header = InputHeader() - self._fieldnames = None - self._finished = None - self._metadata = None - self._options = None - self._protocol_version = None - self._search_results_info = None - self._service = None - - # Internal variables - - self._default_logging_level = self._logger.level - self._record_writer = None - self._records = None - self._allow_empty_input = True - - def __str__(self): - text = ' '.join(chain((type(self).name, str(self.options)), [] if self.fieldnames is None else self.fieldnames)) - return text - - # region Options - - @Option - def logging_configuration(self): - """ **Syntax:** logging_configuration= - - **Description:** Loads an alternative logging configuration file for - a command invocation. The logging configuration file must be in Python - ConfigParser-format. Path names are relative to the app root directory. - - """ - return self._logging_configuration - - @logging_configuration.setter - def logging_configuration(self, value): - self._logger, self._logging_configuration = environment.configure_logging(self.__class__.__name__, value) - - @Option - def logging_level(self): - """ **Syntax:** logging_level=[CRITICAL|ERROR|WARNING|INFO|DEBUG|NOTSET] - - **Description:** Sets the threshold for the logger of this command invocation. Logging messages less severe than - `logging_level` will be ignored. - - """ - return getLevelName(self._logger.getEffectiveLevel()) - - @logging_level.setter - def logging_level(self, value): - if value is None: - value = self._default_logging_level - if isinstance(value, (bytes, str)): - try: - level = _levelNames[value.upper()] - except KeyError: - raise ValueError(f'Unrecognized logging level: {value}') - else: - try: - level = int(value) - except ValueError: - raise ValueError(f'Unrecognized logging level: {value}') - self._logger.setLevel(level) - - def add_field(self, current_record, field_name, field_value): - self._record_writer.custom_fields.add(field_name) - current_record[field_name] = field_value - - def gen_record(self, **record): - self._record_writer.custom_fields |= set(record.keys()) - return record - - record = Option(doc=''' - **Syntax: record= - - **Description:** When `true`, records the interaction between the command and splunkd. Defaults to `false`. - - ''', default=False, validate=Boolean()) - - show_configuration = Option(doc=''' - **Syntax:** show_configuration= - - **Description:** When `true`, reports command configuration as an informational message. Defaults to `false`. - - ''', default=False, validate=Boolean()) - - # endregion - - # region Properties - - @property - def configuration(self): - """ Returns the configuration settings for this command. - - """ - return self._configuration - - @property - def fieldnames(self): - """ Returns the fieldnames specified as argument to this command. - - """ - return self._fieldnames - - @fieldnames.setter - def fieldnames(self, value): - self._fieldnames = value - - @property - def input_header(self): - """ Returns the input header for this command. - - :return: The input header for this command. - :rtype: InputHeader - - """ - warn( - 'SearchCommand.input_header is deprecated and will be removed in a future release. ' - 'Please use SearchCommand.metadata instead.', DeprecationWarning, 2) - return self._input_header - - @property - def logger(self): - """ Returns the logger for this command. - - :return: The logger for this command. - :rtype: - - """ - return self._logger - - @property - def metadata(self): - return self._metadata - - @property - def options(self): - """ Returns the options specified as argument to this command. - - """ - if self._options is None: - self._options = Option.View(self) - return self._options - - @property - def protocol_version(self): - return self._protocol_version - - @property - def search_results_info(self): - """ Returns the search results info for this command invocation. - - The search results info object is created from the search results info file associated with the command - invocation. - - :return: Search results info:const:`None`, if the search results info file associated with the command - invocation is inaccessible. - :rtype: SearchResultsInfo or NoneType - - """ - if self._search_results_info is not None: - return self._search_results_info - - if self._protocol_version == 1: - try: - path = self._input_header['infoPath'] - except KeyError: - return None - else: - assert self._protocol_version == 2 - - try: - dispatch_dir = self._metadata.searchinfo.dispatch_dir - except AttributeError: - return None - - path = os.path.join(dispatch_dir, 'info.csv') - - try: - with io.open(path, 'r') as f: - reader = csv.reader(f, dialect=CsvDialect) - fields = next(reader) - values = next(reader) - except IOError as error: - if error.errno == 2: - self.logger.error(f'Search results info file {json_encode_string(path)} does not exist.') - return - raise - - def convert_field(field): - return (field[1:] if field[0] == '_' else field).replace('.', '_') - - decode = MetadataDecoder().decode - - def convert_value(value): - try: - return decode(value) if len(value) > 0 else value - except ValueError: - return value - - info = ObjectView(dict((convert_field(f_v[0]), convert_value(f_v[1])) for f_v in zip(fields, values))) - - try: - count_map = info.countMap - except AttributeError: - pass - else: - count_map = count_map.split(';') - n = len(count_map) - info.countMap = dict(list(zip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))) - - try: - msg_type = info.msgType - msg_text = info.msg - except AttributeError: - pass - else: - messages = [t_m for t_m in zip(msg_type.split('\n'), msg_text.split('\n')) if t_m[0] or t_m[1]] - info.msg = [Message(message) for message in messages] - del info.msgType - - try: - info.vix_families = ElementTree.fromstring(info.vix_families) - except AttributeError: - pass - - self._search_results_info = info - return info - - @property - def service(self): - """ Returns a Splunk service object for this command invocation or None. - - The service object is created from the Splunkd URI and authentication token passed to the command invocation in - the search results info file. This data is not passed to a command invocation by default. You must request it by - specifying this pair of configuration settings in commands.conf: - - .. code-block:: python - - enableheader = true - requires_srinfo = true - - The :code:`enableheader` setting is :code:`true` by default. Hence, you need not set it. The - :code:`requires_srinfo` setting is false by default. Hence, you must set it. - - :return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both - :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value - of :code:`None` is returned. - - """ - if self._service is not None: - return self._service - - metadata = self._metadata - - if metadata is None: - return None - - try: - searchinfo = self._metadata.searchinfo - except AttributeError: - return None - - splunkd_uri = searchinfo.splunkd_uri - - if splunkd_uri is None: - return None - - uri = urlsplit(splunkd_uri, allow_fragments=False) - - self._service = Service( - scheme=uri.scheme, host=uri.hostname, port=uri.port, app=searchinfo.app, token=searchinfo.session_key) - - return self._service - - # endregion - - # region Methods - - def error_exit(self, error, message=None): - self.write_error(error.message if message is None else message) - self.logger.error('Abnormal exit: %s', error) - exit(1) - - def finish(self): - """ Flushes the output buffer and signals that this command has finished processing data. - - :return: :const:`None` - - """ - self._record_writer.flush(finished=True) - - def flush(self): - """ Flushes the output buffer. - - :return: :const:`None` - - """ - self._record_writer.flush(finished=False) - - def prepare(self): - """ Prepare for execution. - - This method should be overridden in search command classes that wish to examine and update their configuration - or option settings prior to execution. It is called during the getinfo exchange before command metadata is sent - to splunkd. - - :return: :const:`None` - :rtype: NoneType - - """ - - def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True): - """ Process data. - - :param argv: Command line arguments. - :type argv: list or tuple - - :param ifile: Input data file. - :type ifile: file - - :param ofile: Output data file. - :type ofile: file - - :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read - :type allow_empty_input: bool - - :return: :const:`None` - :rtype: NoneType - - """ - - self._allow_empty_input = allow_empty_input - - if len(argv) > 1: - self._process_protocol_v1(argv, ifile, ofile) - else: - self._process_protocol_v2(argv, ifile, ofile) - - def _map_input_header(self): - metadata = self._metadata - searchinfo = metadata.searchinfo - self._input_header.update( - allowStream=None, - infoPath=os.path.join(searchinfo.dispatch_dir, 'info.csv'), - keywords=None, - preview=metadata.preview, - realtime=searchinfo.earliest_time != 0 and searchinfo.latest_time != 0, - search=searchinfo.search, - sid=searchinfo.sid, - splunkVersion=searchinfo.splunk_version, - truncated=None) - - def _map_metadata(self, argv): - source = SearchCommand._MetadataSource(argv, self._input_header, self.search_results_info) - - def _map(metadata_map): - metadata = {} - - for name, value in metadata_map.items(): - if isinstance(value, dict): - value = _map(value) - else: - transform, extract = value - if extract is None: - value = None - else: - value = extract(source) - if not (value is None or transform is None): - value = transform(value) - metadata[name] = value - - return ObjectView(metadata) - - self._metadata = _map(SearchCommand._metadata_map) - - _metadata_map = { - 'action': - (lambda v: 'getinfo' if v == '__GETINFO__' else 'execute' if v == '__EXECUTE__' else None, - lambda s: s.argv[1]), - 'preview': - (bool, lambda s: s.input_header.get('preview')), - 'searchinfo': { - 'app': - (lambda v: v.ppc_app, lambda s: s.search_results_info), - 'args': - (None, lambda s: s.argv), - 'dispatch_dir': - (os.path.dirname, lambda s: s.input_header.get('infoPath')), - 'earliest_time': - (lambda v: float(v.rt_earliest) if len(v.rt_earliest) > 0 else 0.0, lambda s: s.search_results_info), - 'latest_time': - (lambda v: float(v.rt_latest) if len(v.rt_latest) > 0 else 0.0, lambda s: s.search_results_info), - 'owner': - (None, None), - 'raw_args': - (None, lambda s: s.argv), - 'search': - (unquote, lambda s: s.input_header.get('search')), - 'session_key': - (lambda v: v.auth_token, lambda s: s.search_results_info), - 'sid': - (None, lambda s: s.input_header.get('sid')), - 'splunk_version': - (None, lambda s: s.input_header.get('splunkVersion')), - 'splunkd_uri': - (lambda v: v.splunkd_uri, lambda s: s.search_results_info), - 'username': - (lambda v: v.ppc_user, lambda s: s.search_results_info)}} - - _MetadataSource = namedtuple('Source', ('argv', 'input_header', 'search_results_info')) - - def _prepare_protocol_v1(self, argv, ifile, ofile): - - debug = environment.splunklib_logger.debug - - # Provide as much context as possible in advance of parsing the command line and preparing for execution - - self._input_header.read(ifile) - self._protocol_version = 1 - self._map_metadata(argv) - - debug(' metadata=%r, input_header=%r', self._metadata, self._input_header) - - try: - tempfile.tempdir = self._metadata.searchinfo.dispatch_dir - except AttributeError: - raise RuntimeError(f'{self.__class__.__name__}.metadata.searchinfo.dispatch_dir is undefined') - - debug(' tempfile.tempdir=%r', tempfile.tempdir) - - CommandLineParser.parse(self, argv[2:]) - self.prepare() - - if self.record: - self.record = False - - record_argv = [argv[0], argv[1], str(self._options), ' '.join(self.fieldnames)] - ifile, ofile = self._prepare_recording(record_argv, ifile, ofile) - self._record_writer.ofile = ofile - ifile.record(str(self._input_header), '\n\n') - - if self.show_configuration: - self.write_info(self.name + ' command configuration: ' + str(self._configuration)) - - return ifile # wrapped, if self.record is True - - def _prepare_recording(self, argv, ifile, ofile): - - # Create the recordings directory, if it doesn't already exist - - recordings = os.path.join(environment.splunk_home, 'var', 'run', 'splunklib.searchcommands', 'recordings') - - if not os.path.isdir(recordings): - os.makedirs(recordings) - - # Create input/output recorders from ifile and ofile - - recording = os.path.join(recordings, self.__class__.__name__ + '-' + repr(time()) + '.' + self._metadata.action) - ifile = Recorder(recording + '.input', ifile) - ofile = Recorder(recording + '.output', ofile) - - # Archive the dispatch directory--if it exists--so that it can be used as a baseline in mocks) - - dispatch_dir = self._metadata.searchinfo.dispatch_dir - - if dispatch_dir is not None: # __GETINFO__ action does not include a dispatch_dir - root_dir, base_dir = os.path.split(dispatch_dir) - make_archive(recording + '.dispatch_dir', 'gztar', root_dir, base_dir, logger=self.logger) - - # Save a splunk command line because it is useful for developing tests - - with open(recording + '.splunk_cmd', 'wb') as f: - f.write('splunk cmd python '.encode()) - f.write(os.path.basename(argv[0]).encode()) - for arg in islice(argv, 1, len(argv)): - f.write(' '.encode()) - f.write(arg.encode()) - - return ifile, ofile - - def _process_protocol_v1(self, argv, ifile, ofile): - - debug = environment.splunklib_logger.debug - class_name = self.__class__.__name__ - - debug('%s.process started under protocol_version=1', class_name) - self._record_writer = RecordWriterV1(ofile) - - # noinspection PyBroadException - try: - if argv[1] == '__GETINFO__': - - debug('Writing configuration settings') - - ifile = self._prepare_protocol_v1(argv, ifile, ofile) - self._record_writer.write_record(dict( - (n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in - self._configuration.items())) - self.finish() - - elif argv[1] == '__EXECUTE__': - - debug('Executing') - - ifile = self._prepare_protocol_v1(argv, ifile, ofile) - self._records = self._records_protocol_v1 - self._metadata.action = 'execute' - self._execute(ifile, None) - - else: - message = ( - f'Command {self.name} appears to be statically configured for search command protocol version 1 and static ' - 'configuration is unsupported by splunklib.searchcommands. Please ensure that ' - 'default/commands.conf contains this stanza:\n' - f'[{self.name}]\n' - f'filename = {os.path.basename(argv[0])}\n' - 'enableheader = true\n' - 'outputheader = true\n' - 'requires_srinfo = true\n' - 'supports_getinfo = true\n' - 'supports_multivalues = true\n' - 'supports_rawargs = true') - raise RuntimeError(message) - - except (SyntaxError, ValueError) as error: - self.write_error(str(error)) - self.flush() - exit(0) - - except SystemExit: - self.flush() - raise - - except: - self._report_unexpected_error() - self.flush() - exit(1) - - debug('%s.process finished under protocol_version=1', class_name) - - def _protocol_v2_option_parser(self, arg): - """ Determines if an argument is an Option/Value pair, or just a Positional Argument. - Method so different search commands can handle parsing of arguments differently. - - :param arg: A single argument provided to the command from SPL - :type arg: str - - :return: [OptionName, OptionValue] OR [PositionalArgument] - :rtype: List[str] - - """ - return arg.split('=', 1) - - def _process_protocol_v2(self, argv, ifile, ofile): - """ Processes records on the `input stream optionally writing records to the output stream. - - :param ifile: Input file object. - :type ifile: file or InputType - - :param ofile: Output file object. - :type ofile: file or OutputType - - :return: :const:`None` - - """ - debug = environment.splunklib_logger.debug - class_name = self.__class__.__name__ - - debug('%s.process started under protocol_version=2', class_name) - self._protocol_version = 2 - - # Read search command metadata from splunkd - # noinspection PyBroadException - try: - debug('Reading metadata') - metadata, body = self._read_chunk(self._as_binary_stream(ifile)) - - action = getattr(metadata, 'action', None) - - if action != 'getinfo': - raise RuntimeError(f'Expected getinfo action, not {action}') - - if len(body) > 0: - raise RuntimeError('Did not expect data for getinfo action') - - self._metadata = deepcopy(metadata) - - searchinfo = self._metadata.searchinfo - - searchinfo.earliest_time = float(searchinfo.earliest_time) - searchinfo.latest_time = float(searchinfo.latest_time) - searchinfo.search = unquote(searchinfo.search) - - self._map_input_header() - - debug(' metadata=%r, input_header=%r', self._metadata, self._input_header) - - try: - tempfile.tempdir = self._metadata.searchinfo.dispatch_dir - except AttributeError: - raise RuntimeError(f'{class_name}.metadata.searchinfo.dispatch_dir is undefined') - - debug(' tempfile.tempdir=%r', tempfile.tempdir) - except: - self._record_writer = RecordWriterV2(ofile) - self._report_unexpected_error() - self.finish() - exit(1) - - # Write search command configuration for consumption by splunkd - # noinspection PyBroadException - try: - self._record_writer = RecordWriterV2(ofile, getattr(self._metadata.searchinfo, 'maxresultrows', None)) - self.fieldnames = [] - self.options.reset() - - args = self.metadata.searchinfo.args - error_count = 0 - - debug('Parsing arguments') - - if args and isinstance(args, list): - for arg in args: - result = self._protocol_v2_option_parser(arg) - if len(result) == 1: - self.fieldnames.append(str(result[0])) - else: - name, value = result - name = str(name) - try: - option = self.options[name] - except KeyError: - self.write_error(f'Unrecognized option: {name}={value}') - error_count += 1 - continue - try: - option.value = value - except ValueError: - self.write_error(f'Illegal value: {name}={value}') - error_count += 1 - continue - - missing = self.options.get_missing() - - if missing is not None: - if len(missing) == 1: - self.write_error(f'A value for "{missing[0]}" is required') - else: - self.write_error(f'Values for these required options are missing: {", ".join(missing)}') - error_count += 1 - - if error_count > 0: - exit(1) - - debug(' command: %s', str(self)) - - debug('Preparing for execution') - self.prepare() - - if self.record: - - ifile, ofile = self._prepare_recording(argv, ifile, ofile) - self._record_writer.ofile = ofile - - # Record the metadata that initiated this command after removing the record option from args/raw_args - - info = self._metadata.searchinfo - - for attr in 'args', 'raw_args': - setattr(info, attr, [arg for arg in getattr(info, attr) if not arg.startswith('record=')]) - - metadata = MetadataEncoder().encode(self._metadata) - ifile.record('chunked 1.0,', str(len(metadata)), ',0\n', metadata) - - if self.show_configuration: - self.write_info(self.name + ' command configuration: ' + str(self._configuration)) - - debug(' command configuration: %s', self._configuration) - - except SystemExit: - self._record_writer.write_metadata(self._configuration) - self.finish() - raise - except: - self._record_writer.write_metadata(self._configuration) - self._report_unexpected_error() - self.finish() - exit(1) - - self._record_writer.write_metadata(self._configuration) - - # Execute search command on data passing through the pipeline - # noinspection PyBroadException - try: - debug('Executing under protocol_version=2') - self._metadata.action = 'execute' - self._execute(ifile, None) - except SystemExit: - self.finish() - raise - except: - self._report_unexpected_error() - self.finish() - exit(1) - - debug('%s.process completed', class_name) - - def write_debug(self, message, *args): - self._record_writer.write_message('DEBUG', message, *args) - - def write_error(self, message, *args): - self._record_writer.write_message('ERROR', message, *args) - - def write_fatal(self, message, *args): - self._record_writer.write_message('FATAL', message, *args) - - def write_info(self, message, *args): - self._record_writer.write_message('INFO', message, *args) - - def write_warning(self, message, *args): - self._record_writer.write_message('WARN', message, *args) - - def write_metric(self, name, value): - """ Writes a metric that will be added to the search inspector. - - :param name: Name of the metric. - :type name: basestring - - :param value: A 4-tuple containing the value of metric ``name`` where - - value[0] = Elapsed seconds or :const:`None`. - value[1] = Number of invocations or :const:`None`. - value[2] = Input count or :const:`None`. - value[3] = Output count or :const:`None`. - - The :data:`SearchMetric` type provides a convenient encapsulation of ``value``. - The :data:`SearchMetric` type provides a convenient encapsulation of ``value``. - - :return: :const:`None`. - - """ - self._record_writer.write_metric(name, value) - - # P2 [ ] TODO: Support custom inspector values - - @staticmethod - def _decode_list(mv): - return [match.replace('$$', '$') for match in SearchCommand._encoded_value.findall(mv)] - - _encoded_value = re.compile(r'\$(?P(?:\$\$|[^$])*)\$(?:;|$)') # matches a single value in an encoded list - - # Note: Subclasses must override this method so that it can be called - # called as self._execute(ifile, None) - def _execute(self, ifile, process): - """ Default processing loop - - :param ifile: Input file object. - :type ifile: file - - :param process: Bound method to call in processing loop. - :type process: instancemethod - - :return: :const:`None`. - :rtype: NoneType - - """ - if self.protocol_version == 1: - self._record_writer.write_records(process(self._records(ifile))) - self.finish() - else: - assert self._protocol_version == 2 - self._execute_v2(ifile, process) - - @staticmethod - def _as_binary_stream(ifile): - naught = ifile.read(0) - if isinstance(naught, bytes): - return ifile - - try: - return ifile.buffer - except AttributeError as error: - raise RuntimeError(f'Failed to get underlying buffer: {error}') - - @staticmethod - def _read_chunk(istream): - # noinspection PyBroadException - assert isinstance(istream.read(0), bytes), 'Stream must be binary' - - try: - header = istream.readline() - except Exception as error: - raise RuntimeError(f'Failed to read transport header: {error}') - - if not header: - return None - - match = SearchCommand._header.match(ensure_str(header)) - - if match is None: - raise RuntimeError(f'Failed to parse transport header: {header}') - - metadata_length, body_length = match.groups() - metadata_length = int(metadata_length) - body_length = int(body_length) - - try: - metadata = istream.read(metadata_length) - except Exception as error: - raise RuntimeError(f'Failed to read metadata of length {metadata_length}: {error}') - - decoder = MetadataDecoder() - - try: - metadata = decoder.decode(ensure_str(metadata)) - except Exception as error: - raise RuntimeError(f'Failed to parse metadata of length {metadata_length}: {error}') - - # if body_length <= 0: - # return metadata, '' - - body = "" - try: - if body_length > 0: - body = istream.read(body_length) - except Exception as error: - raise RuntimeError(f'Failed to read body of length {body_length}: {error}') - - return metadata, ensure_str(body,errors="replace") - - _header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n') - - def _records_protocol_v1(self, ifile): - return self._read_csv_records(ifile) - - def _read_csv_records(self, ifile): - reader = csv.reader(ifile, dialect=CsvDialect) - - try: - fieldnames = next(reader) - except StopIteration: - return - - mv_fieldnames = dict((name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')) - - if len(mv_fieldnames) == 0: - for values in reader: - yield OrderedDict(list(zip(fieldnames, values))) - return - - for values in reader: - record = OrderedDict() - for fieldname, value in zip(fieldnames, values): - if fieldname.startswith('__mv_'): - if len(value) > 0: - record[mv_fieldnames[fieldname]] = self._decode_list(value) - elif fieldname not in record: - record[fieldname] = value - yield record - - def _execute_v2(self, ifile, process): - istream = self._as_binary_stream(ifile) - - while True: - result = self._read_chunk(istream) - - if not result: - return - - metadata, body = result - action = getattr(metadata, 'action', None) - if action != 'execute': - raise RuntimeError(f'Expected execute action, not {action}') - - self._finished = getattr(metadata, 'finished', False) - self._record_writer.is_flushed = False - self._metadata.update(metadata) - self._execute_chunk_v2(process, result) - - self._record_writer.write_chunk(finished=self._finished) - - def _execute_chunk_v2(self, process, chunk): - metadata, body = chunk - - if len(body) <= 0 and not self._allow_empty_input: - raise ValueError( - "No records found to process. Set allow_empty_input=True in dispatch function to move forward " - "with empty records.") - - records = self._read_csv_records(StringIO(body)) - self._record_writer.write_records(process(records)) - - def _report_unexpected_error(self): - - error_type, error, tb = sys.exc_info() - origin = tb - - while origin.tb_next is not None: - origin = origin.tb_next - - filename = origin.tb_frame.f_code.co_filename - lineno = origin.tb_lineno - message = f'{error_type.__name__} at "{filename}", line {str(lineno)} : {error}' - - environment.splunklib_logger.error(message + '\nTraceback:\n' + ''.join(traceback.format_tb(tb))) - self.write_error(message) - - # endregion - - # region Types - - class ConfigurationSettings: - """ Represents the configuration settings common to all :class:`SearchCommand` classes. - - """ - - def __init__(self, command): - self.command = command - - def __repr__(self): - """ Converts the value of this instance to its string representation. - - The value of this ConfigurationSettings instance is represented as a string of comma-separated - :code:`(name, value)` pairs. - - :return: String representation of this instance - - """ - definitions = type(self).configuration_setting_definitions - settings = [repr((setting.name, setting.__get__(self), setting.supporting_protocols)) for setting in - definitions] - return '[' + ', '.join(settings) + ']' - - def __str__(self): - """ Converts the value of this instance to its string representation. - - The value of this ConfigurationSettings instance is represented as a string of comma-separated - :code:`name=value` pairs. Items with values of :const:`None` are filtered from the list. - - :return: String representation of this instance - - """ - # text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems())) - text = ', '.join([f'{name}={json_encode_string(str(value))}' for (name, value) in self.items()]) - return text - - # region Methods - - @classmethod - def fix_up(cls, command_class): - """ Adjusts and checks this class and its search command class. - - Derived classes typically override this method. It is used by the :decorator:`Configuration` decorator to - fix up the :class:`SearchCommand` class it adorns. This method is overridden by :class:`EventingCommand`, - :class:`GeneratingCommand`, :class:`ReportingCommand`, and :class:`StreamingCommand`, the base types for - all other search commands. - - :param command_class: Command class targeted by this class - - """ - return - - # TODO: Stop looking like a dictionary because we don't obey the semantics - # N.B.: Does not use Python 2 dict copy semantics - def iteritems(self): - definitions = type(self).configuration_setting_definitions - version = self.command.protocol_version - return [name_value1 for name_value1 in [(setting.name, setting.__get__(self)) for setting in - [setting for setting in definitions if - setting.is_supported_by_protocol(version)]] if - name_value1[1] is not None] - - # N.B.: Does not use Python 3 dict view semantics - - items = iteritems - - # endregion - - # endregion - - -SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count')) - - -def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, - allow_empty_input=True): - """ Instantiates and executes a search command class - - This function implements a `conditional script stanza `_ based on the value of - :code:`module_name`:: - - if module_name is None or module_name == '__main__': - # execute command - - Call this function at module scope with :code:`module_name=__name__`, if you would like your module to act as either - a reusable module or a standalone program. Otherwise, if you wish this function to unconditionally instantiate and - execute :code:`command_class`, pass :const:`None` as the value of :code:`module_name`. - - :param command_class: Search command class to instantiate and execute. - :type command_class: type - :param argv: List of arguments to the command. - :type argv: list or tuple - :param input_file: File from which the command will read data. - :type input_file: :code:`file` - :param output_file: File to which the command will write data. - :type output_file: :code:`file` - :param module_name: Name of the module calling :code:`dispatch` or :const:`None`. - :type module_name: :code:`basestring` - :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read - :type allow_empty_input: bool - :returns: :const:`None` - - **Example** - - .. code-block:: python - :linenos: - - #!/usr/bin/env python - from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators - @Configuration() - class SomeStreamingCommand(StreamingCommand): - ... - def stream(records): - ... - dispatch(SomeStreamingCommand, module_name=__name__) - - Dispatches the :code:`SomeStreamingCommand`, if and only if :code:`__name__` is equal to :code:`'__main__'`. - - **Example** - - .. code-block:: python - :linenos: - - from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators - @Configuration() - class SomeStreamingCommand(StreamingCommand): - ... - def stream(records): - ... - dispatch(SomeStreamingCommand) - - Unconditionally dispatches :code:`SomeStreamingCommand`. - - """ - assert issubclass(command_class, SearchCommand) - - if module_name is None or module_name == '__main__': - command_class().process(argv, input_file, output_file, allow_empty_input) +# coding=utf-8 +# +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Absolute imports + +import csv +import io +import os +import re +import sys +import tempfile +import traceback +from collections import namedtuple, OrderedDict +from copy import deepcopy +from io import StringIO +from itertools import chain, islice +from logging import _nameToLevel as _levelNames, getLevelName, getLogger +from shutil import make_archive +from time import time +from urllib.parse import unquote +from urllib.parse import urlsplit +from warnings import warn +from xml.etree import ElementTree +from splunklib.utils import ensure_str + + +# Relative imports +import splunklib +from . import Boolean, Option, environment +from .internals import ( + CommandLineParser, + CsvDialect, + InputHeader, + Message, + MetadataDecoder, + MetadataEncoder, + ObjectView, + Recorder, + RecordWriterV1, + RecordWriterV2, + json_encode_string) +from ..client import Service + + +# ---------------------------------------------------------------------------------------------------------------------- + +# P1 [ ] TODO: Log these issues against ChunkedExternProcessor +# +# 1. Implement requires_preop configuration setting. +# This configuration setting is currently rejected by ChunkedExternProcessor. +# +# 2. Rename type=events as type=eventing for symmetry with type=reporting and type=streaming +# Eventing commands process records on the events pipeline. This change effects ChunkedExternProcessor.cpp, +# eventing_command.py, and generating_command.py. +# +# 3. For consistency with SCPV1, commands.conf should not require filename setting when chunked = true +# The SCPV1 processor uses .py as the default filename. The ChunkedExternProcessor should do the same. + +# P1 [ ] TODO: Verify that ChunkedExternProcessor complains if a streaming_preop has a type other than 'streaming' +# It once looked like sending type='reporting' for the streaming_preop was accepted. + +# ---------------------------------------------------------------------------------------------------------------------- + +# P2 [ ] TODO: Consider bumping None formatting up to Option.Item.__str__ + + +class SearchCommand: + """ Represents a custom search command. + + """ + + def __init__(self): + + # Variables that may be used, but not altered by derived classes + + class_name = self.__class__.__name__ + + self._logger, self._logging_configuration = getLogger(class_name), environment.logging_configuration + + # Variables backing option/property values + + self._configuration = self.ConfigurationSettings(self) + self._input_header = InputHeader() + self._fieldnames = None + self._finished = None + self._metadata = None + self._options = None + self._protocol_version = None + self._search_results_info = None + self._service = None + + # Internal variables + + self._default_logging_level = self._logger.level + self._record_writer = None + self._records = None + self._allow_empty_input = True + + def __str__(self): + text = ' '.join(chain((type(self).name, str(self.options)), [] if self.fieldnames is None else self.fieldnames)) + return text + + # region Options + + @Option + def logging_configuration(self): + """ **Syntax:** logging_configuration= + + **Description:** Loads an alternative logging configuration file for + a command invocation. The logging configuration file must be in Python + ConfigParser-format. Path names are relative to the app root directory. + + """ + return self._logging_configuration + + @logging_configuration.setter + def logging_configuration(self, value): + self._logger, self._logging_configuration = environment.configure_logging(self.__class__.__name__, value) + + @Option + def logging_level(self): + """ **Syntax:** logging_level=[CRITICAL|ERROR|WARNING|INFO|DEBUG|NOTSET] + + **Description:** Sets the threshold for the logger of this command invocation. Logging messages less severe than + `logging_level` will be ignored. + + """ + return getLevelName(self._logger.getEffectiveLevel()) + + @logging_level.setter + def logging_level(self, value): + if value is None: + value = self._default_logging_level + if isinstance(value, (bytes, str)): + try: + level = _levelNames[value.upper()] + except KeyError: + raise ValueError(f'Unrecognized logging level: {value}') + else: + try: + level = int(value) + except ValueError: + raise ValueError(f'Unrecognized logging level: {value}') + self._logger.setLevel(level) + + def add_field(self, current_record, field_name, field_value): + self._record_writer.custom_fields.add(field_name) + current_record[field_name] = field_value + + def gen_record(self, **record): + self._record_writer.custom_fields |= set(record.keys()) + return record + + record = Option(doc=''' + **Syntax: record= + + **Description:** When `true`, records the interaction between the command and splunkd. Defaults to `false`. + + ''', default=False, validate=Boolean()) + + show_configuration = Option(doc=''' + **Syntax:** show_configuration= + + **Description:** When `true`, reports command configuration as an informational message. Defaults to `false`. + + ''', default=False, validate=Boolean()) + + # endregion + + # region Properties + + @property + def configuration(self): + """ Returns the configuration settings for this command. + + """ + return self._configuration + + @property + def fieldnames(self): + """ Returns the fieldnames specified as argument to this command. + + """ + return self._fieldnames + + @fieldnames.setter + def fieldnames(self, value): + self._fieldnames = value + + @property + def input_header(self): + """ Returns the input header for this command. + + :return: The input header for this command. + :rtype: InputHeader + + """ + warn( + 'SearchCommand.input_header is deprecated and will be removed in a future release. ' + 'Please use SearchCommand.metadata instead.', DeprecationWarning, 2) + return self._input_header + + @property + def logger(self): + """ Returns the logger for this command. + + :return: The logger for this command. + :rtype: + + """ + return self._logger + + @property + def metadata(self): + return self._metadata + + @property + def options(self): + """ Returns the options specified as argument to this command. + + """ + if self._options is None: + self._options = Option.View(self) + return self._options + + @property + def protocol_version(self): + return self._protocol_version + + @property + def search_results_info(self): + """ Returns the search results info for this command invocation. + + The search results info object is created from the search results info file associated with the command + invocation. + + :return: Search results info:const:`None`, if the search results info file associated with the command + invocation is inaccessible. + :rtype: SearchResultsInfo or NoneType + + """ + if self._search_results_info is not None: + return self._search_results_info + + if self._protocol_version == 1: + try: + path = self._input_header['infoPath'] + except KeyError: + return None + else: + assert self._protocol_version == 2 + + try: + dispatch_dir = self._metadata.searchinfo.dispatch_dir + except AttributeError: + return None + + path = os.path.join(dispatch_dir, 'info.csv') + + try: + with io.open(path, 'r') as f: + reader = csv.reader(f, dialect=CsvDialect) + fields = next(reader) + values = next(reader) + except IOError as error: + if error.errno == 2: + self.logger.error(f'Search results info file {json_encode_string(path)} does not exist.') + return + raise + + def convert_field(field): + return (field[1:] if field[0] == '_' else field).replace('.', '_') + + decode = MetadataDecoder().decode + + def convert_value(value): + try: + return decode(value) if len(value) > 0 else value + except ValueError: + return value + + info = ObjectView(dict((convert_field(f_v[0]), convert_value(f_v[1])) for f_v in zip(fields, values))) + + try: + count_map = info.countMap + except AttributeError: + pass + else: + count_map = count_map.split(';') + n = len(count_map) + info.countMap = dict(list(zip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))) + + try: + msg_type = info.msgType + msg_text = info.msg + except AttributeError: + pass + else: + messages = [t_m for t_m in zip(msg_type.split('\n'), msg_text.split('\n')) if t_m[0] or t_m[1]] + info.msg = [Message(message) for message in messages] + del info.msgType + + try: + info.vix_families = ElementTree.fromstring(info.vix_families) + except AttributeError: + pass + + self._search_results_info = info + return info + + @property + def service(self): + """ Returns a Splunk service object for this command invocation or None. + + The service object is created from the Splunkd URI and authentication token passed to the command invocation in + the search results info file. This data is not passed to a command invocation by default. You must request it by + specifying this pair of configuration settings in commands.conf: + + .. code-block:: python + + enableheader = true + requires_srinfo = true + + The :code:`enableheader` setting is :code:`true` by default. Hence, you need not set it. The + :code:`requires_srinfo` setting is false by default. Hence, you must set it. + + :return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both + :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value + of :code:`None` is returned. + + """ + if self._service is not None: + return self._service + + metadata = self._metadata + + if metadata is None: + return None + + try: + searchinfo = self._metadata.searchinfo + except AttributeError: + return None + + splunkd_uri = searchinfo.splunkd_uri + + if splunkd_uri is None: + return None + + uri = urlsplit(splunkd_uri, allow_fragments=False) + + self._service = Service( + scheme=uri.scheme, host=uri.hostname, port=uri.port, app=searchinfo.app, token=searchinfo.session_key) + + return self._service + + # endregion + + # region Methods + + def error_exit(self, error, message=None): + self.write_error(error.message if message is None else message) + self.logger.error('Abnormal exit: %s', error) + exit(1) + + def finish(self): + """ Flushes the output buffer and signals that this command has finished processing data. + + :return: :const:`None` + + """ + self._record_writer.flush(finished=True) + + def flush(self): + """ Flushes the output buffer. + + :return: :const:`None` + + """ + self._record_writer.flush(finished=False) + + def prepare(self): + """ Prepare for execution. + + This method should be overridden in search command classes that wish to examine and update their configuration + or option settings prior to execution. It is called during the getinfo exchange before command metadata is sent + to splunkd. + + :return: :const:`None` + :rtype: NoneType + + """ + + def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True): + """ Process data. + + :param argv: Command line arguments. + :type argv: list or tuple + + :param ifile: Input data file. + :type ifile: file + + :param ofile: Output data file. + :type ofile: file + + :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read + :type allow_empty_input: bool + + :return: :const:`None` + :rtype: NoneType + + """ + + self._allow_empty_input = allow_empty_input + + if len(argv) > 1: + self._process_protocol_v1(argv, ifile, ofile) + else: + self._process_protocol_v2(argv, ifile, ofile) + + def _map_input_header(self): + metadata = self._metadata + searchinfo = metadata.searchinfo + self._input_header.update( + allowStream=None, + infoPath=os.path.join(searchinfo.dispatch_dir, 'info.csv'), + keywords=None, + preview=metadata.preview, + realtime=searchinfo.earliest_time != 0 and searchinfo.latest_time != 0, + search=searchinfo.search, + sid=searchinfo.sid, + splunkVersion=searchinfo.splunk_version, + truncated=None) + + def _map_metadata(self, argv): + source = SearchCommand._MetadataSource(argv, self._input_header, self.search_results_info) + + def _map(metadata_map): + metadata = {} + + for name, value in metadata_map.items(): + if isinstance(value, dict): + value = _map(value) + else: + transform, extract = value + if extract is None: + value = None + else: + value = extract(source) + if not (value is None or transform is None): + value = transform(value) + metadata[name] = value + + return ObjectView(metadata) + + self._metadata = _map(SearchCommand._metadata_map) + + _metadata_map = { + 'action': + (lambda v: 'getinfo' if v == '__GETINFO__' else 'execute' if v == '__EXECUTE__' else None, + lambda s: s.argv[1]), + 'preview': + (bool, lambda s: s.input_header.get('preview')), + 'searchinfo': { + 'app': + (lambda v: v.ppc_app, lambda s: s.search_results_info), + 'args': + (None, lambda s: s.argv), + 'dispatch_dir': + (os.path.dirname, lambda s: s.input_header.get('infoPath')), + 'earliest_time': + (lambda v: float(v.rt_earliest) if len(v.rt_earliest) > 0 else 0.0, lambda s: s.search_results_info), + 'latest_time': + (lambda v: float(v.rt_latest) if len(v.rt_latest) > 0 else 0.0, lambda s: s.search_results_info), + 'owner': + (None, None), + 'raw_args': + (None, lambda s: s.argv), + 'search': + (unquote, lambda s: s.input_header.get('search')), + 'session_key': + (lambda v: v.auth_token, lambda s: s.search_results_info), + 'sid': + (None, lambda s: s.input_header.get('sid')), + 'splunk_version': + (None, lambda s: s.input_header.get('splunkVersion')), + 'splunkd_uri': + (lambda v: v.splunkd_uri, lambda s: s.search_results_info), + 'username': + (lambda v: v.ppc_user, lambda s: s.search_results_info)}} + + _MetadataSource = namedtuple('Source', ('argv', 'input_header', 'search_results_info')) + + def _prepare_protocol_v1(self, argv, ifile, ofile): + + debug = environment.splunklib_logger.debug + + # Provide as much context as possible in advance of parsing the command line and preparing for execution + + self._input_header.read(ifile) + self._protocol_version = 1 + self._map_metadata(argv) + + debug(' metadata=%r, input_header=%r', self._metadata, self._input_header) + + try: + tempfile.tempdir = self._metadata.searchinfo.dispatch_dir + except AttributeError: + raise RuntimeError(f'{self.__class__.__name__}.metadata.searchinfo.dispatch_dir is undefined') + + debug(' tempfile.tempdir=%r', tempfile.tempdir) + + CommandLineParser.parse(self, argv[2:]) + self.prepare() + + if self.record: + self.record = False + + record_argv = [argv[0], argv[1], str(self._options), ' '.join(self.fieldnames)] + ifile, ofile = self._prepare_recording(record_argv, ifile, ofile) + self._record_writer.ofile = ofile + ifile.record(str(self._input_header), '\n\n') + + if self.show_configuration: + self.write_info(self.name + ' command configuration: ' + str(self._configuration)) + + return ifile # wrapped, if self.record is True + + def _prepare_recording(self, argv, ifile, ofile): + + # Create the recordings directory, if it doesn't already exist + + recordings = os.path.join(environment.splunk_home, 'var', 'run', 'splunklib.searchcommands', 'recordings') + + if not os.path.isdir(recordings): + os.makedirs(recordings) + + # Create input/output recorders from ifile and ofile + + recording = os.path.join(recordings, self.__class__.__name__ + '-' + repr(time()) + '.' + self._metadata.action) + ifile = Recorder(recording + '.input', ifile) + ofile = Recorder(recording + '.output', ofile) + + # Archive the dispatch directory--if it exists--so that it can be used as a baseline in mocks) + + dispatch_dir = self._metadata.searchinfo.dispatch_dir + + if dispatch_dir is not None: # __GETINFO__ action does not include a dispatch_dir + root_dir, base_dir = os.path.split(dispatch_dir) + make_archive(recording + '.dispatch_dir', 'gztar', root_dir, base_dir, logger=self.logger) + + # Save a splunk command line because it is useful for developing tests + + with open(recording + '.splunk_cmd', 'wb') as f: + f.write('splunk cmd python '.encode()) + f.write(os.path.basename(argv[0]).encode()) + for arg in islice(argv, 1, len(argv)): + f.write(' '.encode()) + f.write(arg.encode()) + + return ifile, ofile + + def _process_protocol_v1(self, argv, ifile, ofile): + + debug = environment.splunklib_logger.debug + class_name = self.__class__.__name__ + + debug('%s.process started under protocol_version=1', class_name) + self._record_writer = RecordWriterV1(ofile) + + # noinspection PyBroadException + try: + if argv[1] == '__GETINFO__': + + debug('Writing configuration settings') + + ifile = self._prepare_protocol_v1(argv, ifile, ofile) + self._record_writer.write_record(dict( + (n, ','.join(v) if isinstance(v, (list, tuple)) else v) for n, v in + self._configuration.items())) + self.finish() + + elif argv[1] == '__EXECUTE__': + + debug('Executing') + + ifile = self._prepare_protocol_v1(argv, ifile, ofile) + self._records = self._records_protocol_v1 + self._metadata.action = 'execute' + self._execute(ifile, None) + + else: + message = ( + f'Command {self.name} appears to be statically configured for search command protocol version 1 and static ' + 'configuration is unsupported by splunklib.searchcommands. Please ensure that ' + 'default/commands.conf contains this stanza:\n' + f'[{self.name}]\n' + f'filename = {os.path.basename(argv[0])}\n' + 'enableheader = true\n' + 'outputheader = true\n' + 'requires_srinfo = true\n' + 'supports_getinfo = true\n' + 'supports_multivalues = true\n' + 'supports_rawargs = true') + raise RuntimeError(message) + + except (SyntaxError, ValueError) as error: + self.write_error(str(error)) + self.flush() + exit(0) + + except SystemExit: + self.flush() + raise + + except: + self._report_unexpected_error() + self.flush() + exit(1) + + debug('%s.process finished under protocol_version=1', class_name) + + def _protocol_v2_option_parser(self, arg): + """ Determines if an argument is an Option/Value pair, or just a Positional Argument. + Method so different search commands can handle parsing of arguments differently. + + :param arg: A single argument provided to the command from SPL + :type arg: str + + :return: [OptionName, OptionValue] OR [PositionalArgument] + :rtype: List[str] + + """ + return arg.split('=', 1) + + def _process_protocol_v2(self, argv, ifile, ofile): + """ Processes records on the `input stream optionally writing records to the output stream. + + :param ifile: Input file object. + :type ifile: file or InputType + + :param ofile: Output file object. + :type ofile: file or OutputType + + :return: :const:`None` + + """ + debug = environment.splunklib_logger.debug + class_name = self.__class__.__name__ + + debug('%s.process started under protocol_version=2', class_name) + self._protocol_version = 2 + + # Read search command metadata from splunkd + # noinspection PyBroadException + try: + debug('Reading metadata') + metadata, body = self._read_chunk(self._as_binary_stream(ifile)) + + action = getattr(metadata, 'action', None) + + if action != 'getinfo': + raise RuntimeError(f'Expected getinfo action, not {action}') + + if len(body) > 0: + raise RuntimeError('Did not expect data for getinfo action') + + self._metadata = deepcopy(metadata) + + searchinfo = self._metadata.searchinfo + + searchinfo.earliest_time = float(searchinfo.earliest_time) + searchinfo.latest_time = float(searchinfo.latest_time) + searchinfo.search = unquote(searchinfo.search) + + self._map_input_header() + + debug(' metadata=%r, input_header=%r', self._metadata, self._input_header) + + try: + tempfile.tempdir = self._metadata.searchinfo.dispatch_dir + except AttributeError: + raise RuntimeError(f'{class_name}.metadata.searchinfo.dispatch_dir is undefined') + + debug(' tempfile.tempdir=%r', tempfile.tempdir) + except: + self._record_writer = RecordWriterV2(ofile) + self._report_unexpected_error() + self.finish() + exit(1) + + # Write search command configuration for consumption by splunkd + # noinspection PyBroadException + try: + self._record_writer = RecordWriterV2(ofile, getattr(self._metadata.searchinfo, 'maxresultrows', None)) + self.fieldnames = [] + self.options.reset() + + args = self.metadata.searchinfo.args + error_count = 0 + + debug('Parsing arguments') + + if args and isinstance(args, list): + for arg in args: + result = self._protocol_v2_option_parser(arg) + if len(result) == 1: + self.fieldnames.append(str(result[0])) + else: + name, value = result + name = str(name) + try: + option = self.options[name] + except KeyError: + self.write_error(f'Unrecognized option: {name}={value}') + error_count += 1 + continue + try: + option.value = value + except ValueError: + self.write_error(f'Illegal value: {name}={value}') + error_count += 1 + continue + + missing = self.options.get_missing() + + if missing is not None: + if len(missing) == 1: + self.write_error(f'A value for "{missing[0]}" is required') + else: + self.write_error(f'Values for these required options are missing: {", ".join(missing)}') + error_count += 1 + + if error_count > 0: + exit(1) + + debug(' command: %s', str(self)) + + debug('Preparing for execution') + self.prepare() + + if self.record: + + ifile, ofile = self._prepare_recording(argv, ifile, ofile) + self._record_writer.ofile = ofile + + # Record the metadata that initiated this command after removing the record option from args/raw_args + + info = self._metadata.searchinfo + + for attr in 'args', 'raw_args': + setattr(info, attr, [arg for arg in getattr(info, attr) if not arg.startswith('record=')]) + + metadata = MetadataEncoder().encode(self._metadata) + ifile.record('chunked 1.0,', str(len(metadata)), ',0\n', metadata) + + if self.show_configuration: + self.write_info(self.name + ' command configuration: ' + str(self._configuration)) + + debug(' command configuration: %s', self._configuration) + + except SystemExit: + self._record_writer.write_metadata(self._configuration) + self.finish() + raise + except: + self._record_writer.write_metadata(self._configuration) + self._report_unexpected_error() + self.finish() + exit(1) + + self._record_writer.write_metadata(self._configuration) + + # Execute search command on data passing through the pipeline + # noinspection PyBroadException + try: + debug('Executing under protocol_version=2') + self._metadata.action = 'execute' + self._execute(ifile, None) + except SystemExit: + self.finish() + raise + except: + self._report_unexpected_error() + self.finish() + exit(1) + + debug('%s.process completed', class_name) + + def write_debug(self, message, *args): + self._record_writer.write_message('DEBUG', message, *args) + + def write_error(self, message, *args): + self._record_writer.write_message('ERROR', message, *args) + + def write_fatal(self, message, *args): + self._record_writer.write_message('FATAL', message, *args) + + def write_info(self, message, *args): + self._record_writer.write_message('INFO', message, *args) + + def write_warning(self, message, *args): + self._record_writer.write_message('WARN', message, *args) + + def write_metric(self, name, value): + """ Writes a metric that will be added to the search inspector. + + :param name: Name of the metric. + :type name: basestring + + :param value: A 4-tuple containing the value of metric ``name`` where + + value[0] = Elapsed seconds or :const:`None`. + value[1] = Number of invocations or :const:`None`. + value[2] = Input count or :const:`None`. + value[3] = Output count or :const:`None`. + + The :data:`SearchMetric` type provides a convenient encapsulation of ``value``. + The :data:`SearchMetric` type provides a convenient encapsulation of ``value``. + + :return: :const:`None`. + + """ + self._record_writer.write_metric(name, value) + + # P2 [ ] TODO: Support custom inspector values + + @staticmethod + def _decode_list(mv): + return [match.replace('$$', '$') for match in SearchCommand._encoded_value.findall(mv)] + + _encoded_value = re.compile(r'\$(?P(?:\$\$|[^$])*)\$(?:;|$)') # matches a single value in an encoded list + + # Note: Subclasses must override this method so that it can be called + # called as self._execute(ifile, None) + def _execute(self, ifile, process): + """ Default processing loop + + :param ifile: Input file object. + :type ifile: file + + :param process: Bound method to call in processing loop. + :type process: instancemethod + + :return: :const:`None`. + :rtype: NoneType + + """ + if self.protocol_version == 1: + self._record_writer.write_records(process(self._records(ifile))) + self.finish() + else: + assert self._protocol_version == 2 + self._execute_v2(ifile, process) + + @staticmethod + def _as_binary_stream(ifile): + naught = ifile.read(0) + if isinstance(naught, bytes): + return ifile + + try: + return ifile.buffer + except AttributeError as error: + raise RuntimeError(f'Failed to get underlying buffer: {error}') + + @staticmethod + def _read_chunk(istream): + # noinspection PyBroadException + assert isinstance(istream.read(0), bytes), 'Stream must be binary' + + try: + header = istream.readline() + except Exception as error: + raise RuntimeError(f'Failed to read transport header: {error}') + + if not header: + return None + + match = SearchCommand._header.match(ensure_str(header)) + + if match is None: + raise RuntimeError(f'Failed to parse transport header: {header}') + + metadata_length, body_length = match.groups() + metadata_length = int(metadata_length) + body_length = int(body_length) + + try: + metadata = istream.read(metadata_length) + except Exception as error: + raise RuntimeError(f'Failed to read metadata of length {metadata_length}: {error}') + + decoder = MetadataDecoder() + + try: + metadata = decoder.decode(ensure_str(metadata)) + except Exception as error: + raise RuntimeError(f'Failed to parse metadata of length {metadata_length}: {error}') + + # if body_length <= 0: + # return metadata, '' + + body = "" + try: + if body_length > 0: + body = istream.read(body_length) + except Exception as error: + raise RuntimeError(f'Failed to read body of length {body_length}: {error}') + + return metadata, ensure_str(body,errors="replace") + + _header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n') + + def _records_protocol_v1(self, ifile): + return self._read_csv_records(ifile) + + def _read_csv_records(self, ifile): + reader = csv.reader(ifile, dialect=CsvDialect) + + try: + fieldnames = next(reader) + except StopIteration: + return + + mv_fieldnames = dict((name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')) + + if len(mv_fieldnames) == 0: + for values in reader: + yield OrderedDict(list(zip(fieldnames, values))) + return + + for values in reader: + record = OrderedDict() + for fieldname, value in zip(fieldnames, values): + if fieldname.startswith('__mv_'): + if len(value) > 0: + record[mv_fieldnames[fieldname]] = self._decode_list(value) + elif fieldname not in record: + record[fieldname] = value + yield record + + def _execute_v2(self, ifile, process): + istream = self._as_binary_stream(ifile) + + while True: + result = self._read_chunk(istream) + + if not result: + return + + metadata, body = result + action = getattr(metadata, 'action', None) + if action != 'execute': + raise RuntimeError(f'Expected execute action, not {action}') + + self._finished = getattr(metadata, 'finished', False) + self._record_writer.is_flushed = False + self._metadata.update(metadata) + self._execute_chunk_v2(process, result) + + self._record_writer.write_chunk(finished=self._finished) + + def _execute_chunk_v2(self, process, chunk): + metadata, body = chunk + + if len(body) <= 0 and not self._allow_empty_input: + raise ValueError( + "No records found to process. Set allow_empty_input=True in dispatch function to move forward " + "with empty records.") + + records = self._read_csv_records(StringIO(body)) + self._record_writer.write_records(process(records)) + + def _report_unexpected_error(self): + + error_type, error, tb = sys.exc_info() + origin = tb + + while origin.tb_next is not None: + origin = origin.tb_next + + filename = origin.tb_frame.f_code.co_filename + lineno = origin.tb_lineno + message = f'{error_type.__name__} at "{filename}", line {str(lineno)} : {error}' + + environment.splunklib_logger.error(message + '\nTraceback:\n' + ''.join(traceback.format_tb(tb))) + self.write_error(message) + + # endregion + + # region Types + + class ConfigurationSettings: + """ Represents the configuration settings common to all :class:`SearchCommand` classes. + + """ + + def __init__(self, command): + self.command = command + + def __repr__(self): + """ Converts the value of this instance to its string representation. + + The value of this ConfigurationSettings instance is represented as a string of comma-separated + :code:`(name, value)` pairs. + + :return: String representation of this instance + + """ + definitions = type(self).configuration_setting_definitions + settings = [repr((setting.name, setting.__get__(self), setting.supporting_protocols)) for setting in + definitions] + return '[' + ', '.join(settings) + ']' + + def __str__(self): + """ Converts the value of this instance to its string representation. + + The value of this ConfigurationSettings instance is represented as a string of comma-separated + :code:`name=value` pairs. Items with values of :const:`None` are filtered from the list. + + :return: String representation of this instance + + """ + # text = ', '.join(imap(lambda (name, value): name + '=' + json_encode_string(unicode(value)), self.iteritems())) + text = ', '.join([f'{name}={json_encode_string(str(value))}' for (name, value) in self.items()]) + return text + + # region Methods + + @classmethod + def fix_up(cls, command_class): + """ Adjusts and checks this class and its search command class. + + Derived classes typically override this method. It is used by the :decorator:`Configuration` decorator to + fix up the :class:`SearchCommand` class it adorns. This method is overridden by :class:`EventingCommand`, + :class:`GeneratingCommand`, :class:`ReportingCommand`, and :class:`StreamingCommand`, the base types for + all other search commands. + + :param command_class: Command class targeted by this class + + """ + return + + # TODO: Stop looking like a dictionary because we don't obey the semantics + # N.B.: Does not use Python 2 dict copy semantics + def iteritems(self): + definitions = type(self).configuration_setting_definitions + version = self.command.protocol_version + return [name_value1 for name_value1 in [(setting.name, setting.__get__(self)) for setting in + [setting for setting in definitions if + setting.is_supported_by_protocol(version)]] if + name_value1[1] is not None] + + # N.B.: Does not use Python 3 dict view semantics + + items = iteritems + + # endregion + + # endregion + + +SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count')) + + +def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, + allow_empty_input=True): + """ Instantiates and executes a search command class + + This function implements a `conditional script stanza `_ based on the value of + :code:`module_name`:: + + if module_name is None or module_name == '__main__': + # execute command + + Call this function at module scope with :code:`module_name=__name__`, if you would like your module to act as either + a reusable module or a standalone program. Otherwise, if you wish this function to unconditionally instantiate and + execute :code:`command_class`, pass :const:`None` as the value of :code:`module_name`. + + :param command_class: Search command class to instantiate and execute. + :type command_class: type + :param argv: List of arguments to the command. + :type argv: list or tuple + :param input_file: File from which the command will read data. + :type input_file: :code:`file` + :param output_file: File to which the command will write data. + :type output_file: :code:`file` + :param module_name: Name of the module calling :code:`dispatch` or :const:`None`. + :type module_name: :code:`basestring` + :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read + :type allow_empty_input: bool + :returns: :const:`None` + + **Example** + + .. code-block:: python + :linenos: + + #!/usr/bin/env python + from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators + @Configuration() + class SomeStreamingCommand(StreamingCommand): + ... + def stream(records): + ... + dispatch(SomeStreamingCommand, module_name=__name__) + + Dispatches the :code:`SomeStreamingCommand`, if and only if :code:`__name__` is equal to :code:`'__main__'`. + + **Example** + + .. code-block:: python + :linenos: + + from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators + @Configuration() + class SomeStreamingCommand(StreamingCommand): + ... + def stream(records): + ... + dispatch(SomeStreamingCommand) + + Unconditionally dispatches :code:`SomeStreamingCommand`. + + """ + assert issubclass(command_class, SearchCommand) + + if module_name is None or module_name == '__main__': + command_class().process(argv, input_file, output_file, allow_empty_input) diff --git a/tests/searchcommands/chunked_data_stream.py b/tests/searchcommands/chunked_data_stream.py index fcd0de7b..02d890af 100644 --- a/tests/searchcommands/chunked_data_stream.py +++ b/tests/searchcommands/chunked_data_stream.py @@ -1,100 +1,100 @@ -import collections -import csv -import io -import json - -import splunklib.searchcommands.internals -from splunklib.utils import ensure_binary, ensure_str - - -class Chunk: - def __init__(self, version, meta, data): - self.version = ensure_str(version) - self.meta = json.loads(meta) - dialect = splunklib.searchcommands.internals.CsvDialect - self.data = csv.DictReader(io.StringIO(data.decode("utf-8")), - dialect=dialect) - - -class ChunkedDataStreamIter(collections.abc.Iterator): - def __init__(self, chunk_stream): - self.chunk_stream = chunk_stream - - def __next__(self): - return self.next() - - def next(self): - try: - return self.chunk_stream.read_chunk() - except EOFError: - raise StopIteration - - -class ChunkedDataStream(collections.abc.Iterable): - def __iter__(self): - return ChunkedDataStreamIter(self) - - def __init__(self, stream): - empty = stream.read(0) - assert isinstance(empty, bytes) - self.stream = stream - - def read_chunk(self): - header = self.stream.readline() - - while len(header) > 0 and header.strip() == b'': - header = self.stream.readline() # Skip empty lines - if len(header) == 0: - raise EOFError - - version, meta, data = header.rstrip().split(b',') - metabytes = self.stream.read(int(meta)) - databytes = self.stream.read(int(data)) - return Chunk(version, metabytes, databytes) - - -def build_chunk(keyval, data=None): - metadata = ensure_binary(json.dumps(keyval)) - data_output = _build_data_csv(data) - return b"chunked 1.0,%d,%d\n%s%s" % (len(metadata), len(data_output), metadata, data_output) - - -def build_empty_searchinfo(): - return { - 'earliest_time': 0, - 'latest_time': 0, - 'search': "", - 'dispatch_dir': "", - 'sid': "", - 'args': [], - 'splunk_version': "42.3.4", - } - - -def build_getinfo_chunk(): - return build_chunk({ - 'action': 'getinfo', - 'preview': False, - 'searchinfo': build_empty_searchinfo()}) - - -def build_data_chunk(data, finished=True): - return build_chunk({'action': 'execute', 'finished': finished}, data) - - -def _build_data_csv(data): - if data is None: - return b'' - if isinstance(data, bytes): - return data - csvout = io.StringIO() - - headers = set() - for datum in data: - headers.update(datum.keys()) - writer = csv.DictWriter(csvout, headers, - dialect=splunklib.searchcommands.internals.CsvDialect) - writer.writeheader() - for datum in data: - writer.writerow(datum) - return ensure_binary(csvout.getvalue()) +import collections +import csv +import io +import json + +import splunklib.searchcommands.internals +from splunklib.utils import ensure_binary, ensure_str + + +class Chunk: + def __init__(self, version, meta, data): + self.version = ensure_str(version) + self.meta = json.loads(meta) + dialect = splunklib.searchcommands.internals.CsvDialect + self.data = csv.DictReader(io.StringIO(data.decode("utf-8")), + dialect=dialect) + + +class ChunkedDataStreamIter(collections.abc.Iterator): + def __init__(self, chunk_stream): + self.chunk_stream = chunk_stream + + def __next__(self): + return self.next() + + def next(self): + try: + return self.chunk_stream.read_chunk() + except EOFError: + raise StopIteration + + +class ChunkedDataStream(collections.abc.Iterable): + def __iter__(self): + return ChunkedDataStreamIter(self) + + def __init__(self, stream): + empty = stream.read(0) + assert isinstance(empty, bytes) + self.stream = stream + + def read_chunk(self): + header = self.stream.readline() + + while len(header) > 0 and header.strip() == b'': + header = self.stream.readline() # Skip empty lines + if len(header) == 0: + raise EOFError + + version, meta, data = header.rstrip().split(b',') + metabytes = self.stream.read(int(meta)) + databytes = self.stream.read(int(data)) + return Chunk(version, metabytes, databytes) + + +def build_chunk(keyval, data=None): + metadata = ensure_binary(json.dumps(keyval)) + data_output = _build_data_csv(data) + return b"chunked 1.0,%d,%d\n%s%s" % (len(metadata), len(data_output), metadata, data_output) + + +def build_empty_searchinfo(): + return { + 'earliest_time': 0, + 'latest_time': 0, + 'search': "", + 'dispatch_dir': "", + 'sid': "", + 'args': [], + 'splunk_version': "42.3.4", + } + + +def build_getinfo_chunk(): + return build_chunk({ + 'action': 'getinfo', + 'preview': False, + 'searchinfo': build_empty_searchinfo()}) + + +def build_data_chunk(data, finished=True): + return build_chunk({'action': 'execute', 'finished': finished}, data) + + +def _build_data_csv(data): + if data is None: + return b'' + if isinstance(data, bytes): + return data + csvout = io.StringIO() + + headers = set() + for datum in data: + headers.update(datum.keys()) + writer = csv.DictWriter(csvout, headers, + dialect=splunklib.searchcommands.internals.CsvDialect) + writer.writeheader() + for datum in data: + writer.writerow(datum) + return ensure_binary(csvout.getvalue()) diff --git a/tests/searchcommands/test_internals_v1.py b/tests/searchcommands/test_internals_v1.py old mode 100644 new mode 100755 index 1e3cf25e..6e41844f --- a/tests/searchcommands/test_internals_v1.py +++ b/tests/searchcommands/test_internals_v1.py @@ -1,343 +1,343 @@ -#!/usr/bin/env python -# -# Copyright © 2011-2024 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from contextlib import closing -from unittest import main, TestCase -import os -from io import StringIO, BytesIO -from functools import reduce -import pytest - -from splunklib.searchcommands.internals import CommandLineParser, InputHeader, RecordWriterV1 -from splunklib.searchcommands.decorators import Configuration, Option -from splunklib.searchcommands.validators import Boolean - -from splunklib.searchcommands.search_command import SearchCommand - - -@pytest.mark.smoke -class TestInternals(TestCase): - def setUp(self): - TestCase.setUp(self) - - def test_command_line_parser(self): - - @Configuration() - class TestCommandLineParserCommand(SearchCommand): - - required_option = Option(validate=Boolean(), require=True) - unnecessary_option = Option(validate=Boolean(), default=True, require=False) - - class ConfigurationSettings(SearchCommand.ConfigurationSettings): - - @classmethod - def fix_up(cls, command_class): pass - - # Command line without fieldnames - - options = ['required_option=true', 'unnecessary_option=false'] - - command = TestCommandLineParserCommand() - CommandLineParser.parse(command, options) - - for option in command.options.values(): - if option.name in ['logging_configuration', 'logging_level', 'record', 'show_configuration']: - self.assertFalse(option.is_set) - continue - self.assertTrue(option.is_set) - - expected = 'testcommandlineparser required_option="t" unnecessary_option="f"' - self.assertEqual(expected, str(command)) - self.assertEqual(command.fieldnames, []) - - # Command line with fieldnames - - fieldnames = ['field_1', 'field_2', 'field_3'] - - command = TestCommandLineParserCommand() - CommandLineParser.parse(command, options + fieldnames) - - for option in command.options.values(): - if option.name in ['logging_configuration', 'logging_level', 'record', 'show_configuration']: - self.assertFalse(option.is_set) - continue - self.assertTrue(option.is_set) - - expected = 'testcommandlineparser required_option="t" unnecessary_option="f" field_1 field_2 field_3' - self.assertEqual(expected, str(command)) - self.assertEqual(command.fieldnames, fieldnames) - - # Command line without any unnecessary options - - command = TestCommandLineParserCommand() - CommandLineParser.parse(command, ['required_option=true'] + fieldnames) - - for option in command.options.values(): - if option.name in ['unnecessary_option', 'logging_configuration', 'logging_level', 'record', - 'show_configuration']: - self.assertFalse(option.is_set) - continue - self.assertTrue(option.is_set) - - expected = 'testcommandlineparser required_option="t" field_1 field_2 field_3' - self.assertEqual(expected, str(command)) - self.assertEqual(command.fieldnames, fieldnames) - - # Command line with missing required options, with or without fieldnames or unnecessary options - - options = ['unnecessary_option=true'] - self.assertRaises(ValueError, CommandLineParser.parse, command, options + fieldnames) - self.assertRaises(ValueError, CommandLineParser.parse, command, options) - self.assertRaises(ValueError, CommandLineParser.parse, command, []) - - # Command line with unrecognized options - - self.assertRaises(ValueError, CommandLineParser.parse, command, - ['unrecognized_option_1=foo', 'unrecognized_option_2=bar']) - - # Command line with a variety of quoted/escaped text options - - @Configuration() - class TestCommandLineParserCommand(SearchCommand): - - text = Option() - - class ConfigurationSettings(SearchCommand.ConfigurationSettings): - - @classmethod - def fix_up(cls, command_class): pass - - strings = [ - r'"foo bar"', - r'"foo/bar"', - r'"foo\\bar"', - r'"""foo bar"""', - r'"\"foo bar\""', - r'Hello\ World!', - r'\"Hello\ World!\"'] - - expected_values = [ - r'foo bar', - r'foo/bar', - r'foo\bar', - r'"foo bar"', - r'"foo bar"', - r'Hello World!', - r'"Hello World!"' - ] - - for string, expected_value in zip(strings, expected_values): - command = TestCommandLineParserCommand() - argv = ['text', '=', string] - CommandLineParser.parse(command, argv) - self.assertEqual(command.text, expected_value) - - for string, expected_value in zip(strings, expected_values): - command = TestCommandLineParserCommand() - argv = [string] - CommandLineParser.parse(command, argv) - self.assertEqual(command.fieldnames[0], expected_value) - - for string, expected_value in zip(strings, expected_values): - command = TestCommandLineParserCommand() - argv = ['text', '=', string] + strings - CommandLineParser.parse(command, argv) - self.assertEqual(command.text, expected_value) - self.assertEqual(command.fieldnames, expected_values) - - strings = [ - 'some\\ string\\', - r'some\ string"', - r'"some string', - r'some"string' - ] - - for string in strings: - command = TestCommandLineParserCommand() - argv = [string] - self.assertRaises(SyntaxError, CommandLineParser.parse, command, argv) - - def test_command_line_parser_unquote(self): - parser = CommandLineParser - - options = [ - r'foo', # unquoted string with no escaped characters - r'fo\o\ b\"a\\r', # unquoted string with some escaped characters - r'"foo"', # quoted string with no special characters - r'"""foobar1"""', # quoted string with quotes escaped like this: "" - r'"\"foobar2\""', # quoted string with quotes escaped like this: \" - r'"foo ""x"" bar"', # quoted string with quotes escaped like this: "" - r'"foo \"x\" bar"', # quoted string with quotes escaped like this: \" - r'"\\foobar"', # quoted string with an escaped backslash - r'"foo \\ bar"', # quoted string with an escaped backslash - r'"foobar\\"', # quoted string with an escaped backslash - r'foo\\\bar', # quoted string with an escaped backslash and an escaped 'b' - r'""', # pair of quotes - r''] # empty string - - expected = [ - r'foo', - r'foo b"a\r', - r'foo', - r'"foobar1"', - r'"foobar2"', - r'foo "x" bar', - r'foo "x" bar', - '\\foobar', - r'foo \ bar', - 'foobar\\', - r'foo\bar', - r'', - r''] - - # Command line with an assortment of string values - - self.assertEqual(expected[-4], parser.unquote(options[-4])) - - for i in range(0, len(options)): - self.assertEqual(expected[i], parser.unquote(options[i])) - - self.assertRaises(SyntaxError, parser.unquote, '"') - self.assertRaises(SyntaxError, parser.unquote, '"foo') - self.assertRaises(SyntaxError, parser.unquote, 'foo"') - self.assertRaises(SyntaxError, parser.unquote, 'foo\\') - - def test_input_header(self): - - # No items - - input_header = InputHeader() - - with closing(StringIO('\r\n')) as input_file: - input_header.read(input_file) - - self.assertEqual(len(input_header), 0) - - # One unnamed single-line item (same as no items) - - input_header = InputHeader() - - with closing(StringIO('this%20is%20an%20unnamed%20single-line%20item\n\n')) as input_file: - input_header.read(input_file) - - self.assertEqual(len(input_header), 0) - - input_header = InputHeader() - - with closing(StringIO('this%20is%20an%20unnamed\nmulti-\nline%20item\n\n')) as input_file: - input_header.read(input_file) - - self.assertEqual(len(input_header), 0) - - # One named single-line item - - input_header = InputHeader() - - with closing(StringIO('Foo:this%20is%20a%20single-line%20item\n\n')) as input_file: - input_header.read(input_file) - - self.assertEqual(len(input_header), 1) - self.assertEqual(input_header['Foo'], 'this is a single-line item') - - input_header = InputHeader() - - with closing(StringIO('Bar:this is a\nmulti-\nline item\n\n')) as input_file: - input_header.read(input_file) - - self.assertEqual(len(input_header), 1) - self.assertEqual(input_header['Bar'], 'this is a\nmulti-\nline item') - - # The infoPath item (which is the path to a file that we open for reads) - - input_header = InputHeader() - - with closing(StringIO('infoPath:non-existent.csv\n\n')) as input_file: - input_header.read(input_file) - - self.assertEqual(len(input_header), 1) - self.assertEqual(input_header['infoPath'], 'non-existent.csv') - - # Set of named items - - collection = { - 'word_list': 'hello\nworld\n!', - 'word_1': 'hello', - 'word_2': 'world', - 'word_3': '!', - 'sentence': 'hello world!'} - - input_header = InputHeader() - text = reduce(lambda value, item: value + f'{item[0]}:{item[1]}\n', collection.items(), '') + '\n' - - with closing(StringIO(text)) as input_file: - input_header.read(input_file) - - self.assertDictEqual(input_header, collection) - - # Set of named items with an unnamed item at the beginning (the only place that an unnamed item can appear) - - with closing(StringIO('unnamed item\n' + text)) as input_file: - input_header.read(input_file) - - self.assertDictEqual(input_header, collection) - - # Test iterators, indirectly through items, keys, and values - - self.assertEqual(sorted(input_header.items()), sorted(collection.items())) - self.assertEqual(sorted(input_header.keys()), sorted(collection.keys())) - self.assertEqual(sorted(input_header.values()), sorted(collection.values())) - - def test_messages_header(self): - - @Configuration() - class TestMessagesHeaderCommand(SearchCommand): - class ConfigurationSettings(SearchCommand.ConfigurationSettings): - - @classmethod - def fix_up(cls, command_class): pass - - command = TestMessagesHeaderCommand() - command._protocol_version = 1 - output_buffer = BytesIO() - command._record_writer = RecordWriterV1(output_buffer) - - messages = [ - (command.write_debug, 'debug_message'), - (command.write_error, 'error_message'), - (command.write_fatal, 'fatal_message'), - (command.write_info, 'info_message'), - (command.write_warning, 'warning_message')] - - for write, message in messages: - write(message) - - command.finish() - - expected = ( - 'debug_message=debug_message\r\n' - 'error_message=error_message\r\n' - 'error_message=fatal_message\r\n' - 'info_message=info_message\r\n' - 'warn_message=warning_message\r\n' - '\r\n') - - self.assertEqual(output_buffer.getvalue().decode('utf-8'), expected) - - _package_path = os.path.dirname(__file__) - - -if __name__ == "__main__": - main() +#!/usr/bin/env python +# +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from contextlib import closing +from unittest import main, TestCase +import os +from io import StringIO, BytesIO +from functools import reduce +import pytest + +from splunklib.searchcommands.internals import CommandLineParser, InputHeader, RecordWriterV1 +from splunklib.searchcommands.decorators import Configuration, Option +from splunklib.searchcommands.validators import Boolean + +from splunklib.searchcommands.search_command import SearchCommand + + +@pytest.mark.smoke +class TestInternals(TestCase): + def setUp(self): + TestCase.setUp(self) + + def test_command_line_parser(self): + + @Configuration() + class TestCommandLineParserCommand(SearchCommand): + + required_option = Option(validate=Boolean(), require=True) + unnecessary_option = Option(validate=Boolean(), default=True, require=False) + + class ConfigurationSettings(SearchCommand.ConfigurationSettings): + + @classmethod + def fix_up(cls, command_class): pass + + # Command line without fieldnames + + options = ['required_option=true', 'unnecessary_option=false'] + + command = TestCommandLineParserCommand() + CommandLineParser.parse(command, options) + + for option in command.options.values(): + if option.name in ['logging_configuration', 'logging_level', 'record', 'show_configuration']: + self.assertFalse(option.is_set) + continue + self.assertTrue(option.is_set) + + expected = 'testcommandlineparser required_option="t" unnecessary_option="f"' + self.assertEqual(expected, str(command)) + self.assertEqual(command.fieldnames, []) + + # Command line with fieldnames + + fieldnames = ['field_1', 'field_2', 'field_3'] + + command = TestCommandLineParserCommand() + CommandLineParser.parse(command, options + fieldnames) + + for option in command.options.values(): + if option.name in ['logging_configuration', 'logging_level', 'record', 'show_configuration']: + self.assertFalse(option.is_set) + continue + self.assertTrue(option.is_set) + + expected = 'testcommandlineparser required_option="t" unnecessary_option="f" field_1 field_2 field_3' + self.assertEqual(expected, str(command)) + self.assertEqual(command.fieldnames, fieldnames) + + # Command line without any unnecessary options + + command = TestCommandLineParserCommand() + CommandLineParser.parse(command, ['required_option=true'] + fieldnames) + + for option in command.options.values(): + if option.name in ['unnecessary_option', 'logging_configuration', 'logging_level', 'record', + 'show_configuration']: + self.assertFalse(option.is_set) + continue + self.assertTrue(option.is_set) + + expected = 'testcommandlineparser required_option="t" field_1 field_2 field_3' + self.assertEqual(expected, str(command)) + self.assertEqual(command.fieldnames, fieldnames) + + # Command line with missing required options, with or without fieldnames or unnecessary options + + options = ['unnecessary_option=true'] + self.assertRaises(ValueError, CommandLineParser.parse, command, options + fieldnames) + self.assertRaises(ValueError, CommandLineParser.parse, command, options) + self.assertRaises(ValueError, CommandLineParser.parse, command, []) + + # Command line with unrecognized options + + self.assertRaises(ValueError, CommandLineParser.parse, command, + ['unrecognized_option_1=foo', 'unrecognized_option_2=bar']) + + # Command line with a variety of quoted/escaped text options + + @Configuration() + class TestCommandLineParserCommand(SearchCommand): + + text = Option() + + class ConfigurationSettings(SearchCommand.ConfigurationSettings): + + @classmethod + def fix_up(cls, command_class): pass + + strings = [ + r'"foo bar"', + r'"foo/bar"', + r'"foo\\bar"', + r'"""foo bar"""', + r'"\"foo bar\""', + r'Hello\ World!', + r'\"Hello\ World!\"'] + + expected_values = [ + r'foo bar', + r'foo/bar', + r'foo\bar', + r'"foo bar"', + r'"foo bar"', + r'Hello World!', + r'"Hello World!"' + ] + + for string, expected_value in zip(strings, expected_values): + command = TestCommandLineParserCommand() + argv = ['text', '=', string] + CommandLineParser.parse(command, argv) + self.assertEqual(command.text, expected_value) + + for string, expected_value in zip(strings, expected_values): + command = TestCommandLineParserCommand() + argv = [string] + CommandLineParser.parse(command, argv) + self.assertEqual(command.fieldnames[0], expected_value) + + for string, expected_value in zip(strings, expected_values): + command = TestCommandLineParserCommand() + argv = ['text', '=', string] + strings + CommandLineParser.parse(command, argv) + self.assertEqual(command.text, expected_value) + self.assertEqual(command.fieldnames, expected_values) + + strings = [ + 'some\\ string\\', + r'some\ string"', + r'"some string', + r'some"string' + ] + + for string in strings: + command = TestCommandLineParserCommand() + argv = [string] + self.assertRaises(SyntaxError, CommandLineParser.parse, command, argv) + + def test_command_line_parser_unquote(self): + parser = CommandLineParser + + options = [ + r'foo', # unquoted string with no escaped characters + r'fo\o\ b\"a\\r', # unquoted string with some escaped characters + r'"foo"', # quoted string with no special characters + r'"""foobar1"""', # quoted string with quotes escaped like this: "" + r'"\"foobar2\""', # quoted string with quotes escaped like this: \" + r'"foo ""x"" bar"', # quoted string with quotes escaped like this: "" + r'"foo \"x\" bar"', # quoted string with quotes escaped like this: \" + r'"\\foobar"', # quoted string with an escaped backslash + r'"foo \\ bar"', # quoted string with an escaped backslash + r'"foobar\\"', # quoted string with an escaped backslash + r'foo\\\bar', # quoted string with an escaped backslash and an escaped 'b' + r'""', # pair of quotes + r''] # empty string + + expected = [ + r'foo', + r'foo b"a\r', + r'foo', + r'"foobar1"', + r'"foobar2"', + r'foo "x" bar', + r'foo "x" bar', + '\\foobar', + r'foo \ bar', + 'foobar\\', + r'foo\bar', + r'', + r''] + + # Command line with an assortment of string values + + self.assertEqual(expected[-4], parser.unquote(options[-4])) + + for i in range(0, len(options)): + self.assertEqual(expected[i], parser.unquote(options[i])) + + self.assertRaises(SyntaxError, parser.unquote, '"') + self.assertRaises(SyntaxError, parser.unquote, '"foo') + self.assertRaises(SyntaxError, parser.unquote, 'foo"') + self.assertRaises(SyntaxError, parser.unquote, 'foo\\') + + def test_input_header(self): + + # No items + + input_header = InputHeader() + + with closing(StringIO('\r\n')) as input_file: + input_header.read(input_file) + + self.assertEqual(len(input_header), 0) + + # One unnamed single-line item (same as no items) + + input_header = InputHeader() + + with closing(StringIO('this%20is%20an%20unnamed%20single-line%20item\n\n')) as input_file: + input_header.read(input_file) + + self.assertEqual(len(input_header), 0) + + input_header = InputHeader() + + with closing(StringIO('this%20is%20an%20unnamed\nmulti-\nline%20item\n\n')) as input_file: + input_header.read(input_file) + + self.assertEqual(len(input_header), 0) + + # One named single-line item + + input_header = InputHeader() + + with closing(StringIO('Foo:this%20is%20a%20single-line%20item\n\n')) as input_file: + input_header.read(input_file) + + self.assertEqual(len(input_header), 1) + self.assertEqual(input_header['Foo'], 'this is a single-line item') + + input_header = InputHeader() + + with closing(StringIO('Bar:this is a\nmulti-\nline item\n\n')) as input_file: + input_header.read(input_file) + + self.assertEqual(len(input_header), 1) + self.assertEqual(input_header['Bar'], 'this is a\nmulti-\nline item') + + # The infoPath item (which is the path to a file that we open for reads) + + input_header = InputHeader() + + with closing(StringIO('infoPath:non-existent.csv\n\n')) as input_file: + input_header.read(input_file) + + self.assertEqual(len(input_header), 1) + self.assertEqual(input_header['infoPath'], 'non-existent.csv') + + # Set of named items + + collection = { + 'word_list': 'hello\nworld\n!', + 'word_1': 'hello', + 'word_2': 'world', + 'word_3': '!', + 'sentence': 'hello world!'} + + input_header = InputHeader() + text = reduce(lambda value, item: value + f'{item[0]}:{item[1]}\n', collection.items(), '') + '\n' + + with closing(StringIO(text)) as input_file: + input_header.read(input_file) + + self.assertDictEqual(input_header, collection) + + # Set of named items with an unnamed item at the beginning (the only place that an unnamed item can appear) + + with closing(StringIO('unnamed item\n' + text)) as input_file: + input_header.read(input_file) + + self.assertDictEqual(input_header, collection) + + # Test iterators, indirectly through items, keys, and values + + self.assertEqual(sorted(input_header.items()), sorted(collection.items())) + self.assertEqual(sorted(input_header.keys()), sorted(collection.keys())) + self.assertEqual(sorted(input_header.values()), sorted(collection.values())) + + def test_messages_header(self): + + @Configuration() + class TestMessagesHeaderCommand(SearchCommand): + class ConfigurationSettings(SearchCommand.ConfigurationSettings): + + @classmethod + def fix_up(cls, command_class): pass + + command = TestMessagesHeaderCommand() + command._protocol_version = 1 + output_buffer = BytesIO() + command._record_writer = RecordWriterV1(output_buffer) + + messages = [ + (command.write_debug, 'debug_message'), + (command.write_error, 'error_message'), + (command.write_fatal, 'fatal_message'), + (command.write_info, 'info_message'), + (command.write_warning, 'warning_message')] + + for write, message in messages: + write(message) + + command.finish() + + expected = ( + 'debug_message=debug_message\r\n' + 'error_message=error_message\r\n' + 'error_message=fatal_message\r\n' + 'info_message=info_message\r\n' + 'warn_message=warning_message\r\n' + '\r\n') + + self.assertEqual(output_buffer.getvalue().decode('utf-8'), expected) + + _package_path = os.path.dirname(__file__) + + +if __name__ == "__main__": + main() diff --git a/tests/test_binding.py b/tests/test_binding.py old mode 100644 new mode 100755 index 9d4dd4b8..fe14c259 --- a/tests/test_binding.py +++ b/tests/test_binding.py @@ -1,975 +1,975 @@ -#!/usr/bin/env python -# -# Copyright © 2011-2024 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from http import server as BaseHTTPServer -from io import BytesIO, StringIO -from threading import Thread -from urllib.request import Request, urlopen - -from xml.etree.ElementTree import XML - -import json -import logging -from tests import testlib -import unittest -import socket -import ssl - -import splunklib -from splunklib import binding -from splunklib.binding import HTTPError, AuthenticationError, UrlEncoded -from splunklib import data -from splunklib.utils import ensure_str - -import pytest - -# splunkd endpoint paths -PATH_USERS = "authentication/users/" - -# XML Namespaces -NAMESPACE_ATOM = "http://www.w3.org/2005/Atom" -NAMESPACE_REST = "http://dev.splunk.com/ns/rest" -NAMESPACE_OPENSEARCH = "http://a9.com/-/spec/opensearch/1.1" - -# XML Extended Name Fragments -XNAMEF_ATOM = "{%s}%%s" % NAMESPACE_ATOM -XNAMEF_REST = "{%s}%%s" % NAMESPACE_REST -XNAMEF_OPENSEARCH = "{%s}%%s" % NAMESPACE_OPENSEARCH - -# XML Extended Names -XNAME_AUTHOR = XNAMEF_ATOM % "author" -XNAME_ENTRY = XNAMEF_ATOM % "entry" -XNAME_FEED = XNAMEF_ATOM % "feed" -XNAME_ID = XNAMEF_ATOM % "id" -XNAME_TITLE = XNAMEF_ATOM % "title" - - -def load(response): - return data.load(response.body.read()) - - -class BindingTestCase(unittest.TestCase): - context = None - - def setUp(self): - logging.info("%s", self.__class__.__name__) - self.opts = testlib.parse([], {}, ".env") - self.context = binding.connect(**self.opts.kwargs) - logging.debug("Connected to splunkd.") - - -class TestResponseReader(BindingTestCase): - def test_empty(self): - response = binding.ResponseReader(BytesIO(b"")) - self.assertTrue(response.empty) - self.assertEqual(response.peek(10), b"") - self.assertEqual(response.read(10), b"") - - arr = bytearray(10) - self.assertEqual(response.readinto(arr), 0) - self.assertEqual(arr, bytearray(10)) - self.assertTrue(response.empty) - - def test_read_past_end(self): - txt = b"abcd" - response = binding.ResponseReader(BytesIO(txt)) - self.assertFalse(response.empty) - self.assertEqual(response.peek(10), txt) - self.assertEqual(response.read(10), txt) - self.assertTrue(response.empty) - self.assertEqual(response.peek(10), b"") - self.assertEqual(response.read(10), b"") - - def test_read_partial(self): - txt = b"This is a test of the emergency broadcasting system." - response = binding.ResponseReader(BytesIO(txt)) - self.assertEqual(response.peek(5), txt[:5]) - self.assertFalse(response.empty) - self.assertEqual(response.read(), txt) - self.assertTrue(response.empty) - self.assertEqual(response.read(), b'') - - def test_readable(self): - txt = "abcd" - response = binding.ResponseReader(StringIO(txt)) - self.assertTrue(response.readable()) - - def test_readinto_bytearray(self): - txt = b"Checking readinto works as expected" - response = binding.ResponseReader(BytesIO(txt)) - arr = bytearray(10) - self.assertEqual(response.readinto(arr), 10) - self.assertEqual(arr[:10], b"Checking r") - self.assertEqual(response.readinto(arr), 10) - self.assertEqual(arr[:10], b"eadinto wo") - self.assertEqual(response.readinto(arr), 10) - self.assertEqual(arr[:10], b"rks as exp") - self.assertEqual(response.readinto(arr), 5) - self.assertEqual(arr[:5], b"ected") - self.assertTrue(response.empty) - - def test_readinto_memoryview(self): - txt = b"Checking readinto works as expected" - response = binding.ResponseReader(BytesIO(txt)) - arr = bytearray(10) - mv = memoryview(arr) - self.assertEqual(response.readinto(mv), 10) - self.assertEqual(arr[:10], b"Checking r") - self.assertEqual(response.readinto(mv), 10) - self.assertEqual(arr[:10], b"eadinto wo") - self.assertEqual(response.readinto(mv), 10) - self.assertEqual(arr[:10], b"rks as exp") - self.assertEqual(response.readinto(mv), 5) - self.assertEqual(arr[:5], b"ected") - self.assertTrue(response.empty) - - -class TestUrlEncoded(BindingTestCase): - def test_idempotent(self): - a = UrlEncoded('abc') - self.assertEqual(a, UrlEncoded(a)) - - def test_append(self): - self.assertEqual(UrlEncoded('a') + UrlEncoded('b'), - UrlEncoded('ab')) - - def test_append_string(self): - self.assertEqual(UrlEncoded('a') + '%', - UrlEncoded('a%')) - - def test_append_to_string(self): - self.assertEqual('%' + UrlEncoded('a'), - UrlEncoded('%a')) - - def test_interpolation_fails(self): - self.assertRaises(TypeError, lambda: UrlEncoded('%s') % 'boris') - - def test_chars(self): - for char, code in [(' ', '%20'), - ('"', '%22'), - ('%', '%25')]: - self.assertEqual(UrlEncoded(char), - UrlEncoded(code, skip_encode=True)) - - def test_repr(self): - self.assertEqual(repr(UrlEncoded('% %')), "UrlEncoded('% %')") - - -class TestAuthority(unittest.TestCase): - def test_authority_default(self): - self.assertEqual(binding._authority(), - "https://localhost:8089") - - def test_ipv4_host(self): - self.assertEqual( - binding._authority( - host="splunk.utopia.net"), - "https://splunk.utopia.net:8089") - - def test_ipv6_host(self): - self.assertEqual( - binding._authority( - host="2001:0db8:85a3:0000:0000:8a2e:0370:7334"), - "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089") - - def test_ipv6_host_enclosed(self): - self.assertEqual( - binding._authority( - host="[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"), - "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089") - - def test_all_fields(self): - self.assertEqual( - binding._authority( - scheme="http", - host="splunk.utopia.net", - port="471"), - "http://splunk.utopia.net:471") - - -class TestUserManipulation(BindingTestCase): - def setUp(self): - BindingTestCase.setUp(self) - self.username = testlib.tmpname() - self.password = "changeme!" - self.roles = "power" - - # Delete user if it exists already - try: - response = self.context.delete(PATH_USERS + self.username) - self.assertEqual(response.status, 200) - except HTTPError as e: - self.assertTrue(e.status in [400, 500]) - - def tearDown(self): - BindingTestCase.tearDown(self) - try: - self.context.delete(PATH_USERS + self.username) - except HTTPError as e: - if e.status not in [400, 500]: - raise - - def test_user_without_role_fails(self): - self.assertRaises(binding.HTTPError, - self.context.post, - PATH_USERS, name=self.username, - password=self.password) - - def test_create_user(self): - response = self.context.post( - PATH_USERS, name=self.username, - password=self.password, roles=self.roles) - self.assertEqual(response.status, 201) - - response = self.context.get(PATH_USERS + self.username) - entry = load(response).feed.entry - self.assertEqual(entry.title, self.username) - - def test_update_user(self): - self.test_create_user() - response = self.context.post( - PATH_USERS + self.username, - password=self.password, - roles=self.roles, - defaultApp="search", - realname="Renzo", - email="email.me@now.com") - self.assertEqual(response.status, 200) - - response = self.context.get(PATH_USERS + self.username) - self.assertEqual(response.status, 200) - entry = load(response).feed.entry - self.assertEqual(entry.title, self.username) - self.assertEqual(entry.content.defaultApp, "search") - self.assertEqual(entry.content.realname, "Renzo") - self.assertEqual(entry.content.email, "email.me@now.com") - - def test_post_with_body_behaves(self): - self.test_create_user() - response = self.context.post( - PATH_USERS + self.username, - body="defaultApp=search", - ) - self.assertEqual(response.status, 200) - - def test_post_with_get_arguments_to_receivers_stream(self): - text = 'Hello, world!' - response = self.context.post( - '/services/receivers/simple', - headers=[('x-splunk-input-mode', 'streaming')], - source='sdk', sourcetype='sdk_test', - body=text - ) - self.assertEqual(response.status, 200) - - -class TestSocket(BindingTestCase): - def test_socket(self): - socket = self.context.connect() - socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) - socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) - socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) - socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) - socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) - socket.write("\r\n".encode('utf-8')) - socket.close() - - # Sockets take bytes not strings - # - # def test_unicode_socket(self): - # socket = self.context.connect() - # socket.write(u"POST %s HTTP/1.1\r\n" %\ - # self.context._abspath("some/path/to/post/to")) - # socket.write(u"Host: %s:%s\r\n" %\ - # (self.context.host, self.context.port)) - # socket.write(u"Accept-Encoding: identity\r\n") - # socket.write((u"Authorization: %s\r\n" %\ - # self.context.token).encode('utf-8')) - # socket.write(u"X-Splunk-Input-Mode: Streaming\r\n") - # socket.write("\r\n") - # socket.close() - - def test_socket_gethostbyname(self): - self.assertTrue(self.context.connect()) - self.context.host = socket.gethostbyname(self.context.host) - self.assertTrue(self.context.connect()) - - -class TestUnicodeConnect(BindingTestCase): - def test_unicode_connect(self): - opts = self.opts.kwargs.copy() - opts['host'] = str(opts['host']) - context = binding.connect(**opts) - # Just check to make sure the service is alive - response = context.get("/services") - self.assertEqual(response.status, 200) - - -@pytest.mark.smoke -class TestAutologin(BindingTestCase): - def test_with_autologin(self): - self.context.autologin = True - self.assertEqual(self.context.get("/services").status, 200) - self.context.logout() - self.assertEqual(self.context.get("/services").status, 200) - - def test_without_autologin(self): - self.context.autologin = False - self.assertEqual(self.context.get("/services").status, 200) - self.context.logout() - self.assertRaises(AuthenticationError, - self.context.get, "/services") - - -class TestAbspath(BindingTestCase): - def setUp(self): - BindingTestCase.setUp(self) - self.kwargs = self.opts.kwargs.copy() - if 'app' in self.kwargs: del self.kwargs['app'] - if 'owner' in self.kwargs: del self.kwargs['owner'] - - def test_default(self): - path = self.context._abspath("foo", owner=None, app=None) - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/services/foo") - - def test_with_owner(self): - path = self.context._abspath("foo", owner="me", app=None) - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me/system/foo") - - def test_with_app(self): - path = self.context._abspath("foo", owner=None, app="MyApp") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") - - def test_with_both(self): - path = self.context._abspath("foo", owner="me", app="MyApp") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me/MyApp/foo") - - def test_user_sharing(self): - path = self.context._abspath("foo", owner="me", app="MyApp", sharing="user") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me/MyApp/foo") - - def test_sharing_app(self): - path = self.context._abspath("foo", owner="me", app="MyApp", sharing="app") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") - - def test_sharing_global(self): - path = self.context._abspath("foo", owner="me", app="MyApp", sharing="global") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") - - def test_sharing_system(self): - path = self.context._abspath("foo bar", owner="me", app="MyApp", sharing="system") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/system/foo%20bar") - - def test_url_forbidden_characters(self): - path = self.context._abspath('/a/b c/d') - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, '/a/b%20c/d') - - def test_context_defaults(self): - context = binding.connect(**self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/services/foo") - - def test_context_with_owner(self): - context = binding.connect(owner="me", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me/system/foo") - - def test_context_with_app(self): - context = binding.connect(app="MyApp", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") - - def test_context_with_both(self): - context = binding.connect(owner="me", app="MyApp", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me/MyApp/foo") - - def test_context_with_user_sharing(self): - context = binding.connect( - owner="me", app="MyApp", sharing="user", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me/MyApp/foo") - - def test_context_with_app_sharing(self): - context = binding.connect( - owner="me", app="MyApp", sharing="app", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") - - def test_context_with_global_sharing(self): - context = binding.connect( - owner="me", app="MyApp", sharing="global", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") - - def test_context_with_system_sharing(self): - context = binding.connect( - owner="me", app="MyApp", sharing="system", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/nobody/system/foo") - - def test_context_with_owner_as_email(self): - context = binding.connect(owner="me@me.com", **self.kwargs) - path = context._abspath("foo") - self.assertTrue(isinstance(path, UrlEncoded)) - self.assertEqual(path, "/servicesNS/me%40me.com/system/foo") - self.assertEqual(path, UrlEncoded("/servicesNS/me@me.com/system/foo")) - - -# An urllib2 based HTTP request handler, used to test the binding layers -# support for pluggable request handlers. -def urllib2_handler(url, message, **kwargs): - method = message['method'].lower() - data = message.get('body', b"") if method == 'post' else None - headers = dict(message.get('headers', [])) - req = Request(url, data, headers) - try: - response = urlopen(req, context=ssl._create_unverified_context()) - except HTTPError as response: - pass # Propagate HTTP errors via the returned response message - return { - 'status': response.code, - 'reason': response.msg, - 'headers': dict(response.info()), - 'body': BytesIO(response.read()) - } - - -def isatom(body): - """Answers if the given response body looks like ATOM.""" - root = XML(body) - return \ - root.tag == XNAME_FEED and \ - root.find(XNAME_AUTHOR) is not None and \ - root.find(XNAME_ID) is not None and \ - root.find(XNAME_TITLE) is not None - - -class TestPluggableHTTP(testlib.SDKTestCase): - # Verify pluggable HTTP reqeust handlers. - def test_handlers(self): - paths = ["/services", "authentication/users", - "search/jobs"] - handlers = [binding.handler(), # default handler - urllib2_handler] - for handler in handlers: - logging.debug("Connecting with handler %s", handler) - context = binding.connect( - handler=handler, - **self.opts.kwargs) - for path in paths: - body = context.get(path).body.read() - self.assertTrue(isatom(body)) - - -def urllib2_insert_cookie_handler(url, message, **kwargs): - method = message['method'].lower() - data = message.get('body', b"") if method == 'post' else None - headers = dict(message.get('headers', [])) - req = Request(url, data, headers) - try: - response = urlopen(req, context=ssl._create_unverified_context()) - except HTTPError as response: - pass # Propagate HTTP errors via the returned response message - - # Mimic the insertion of 3rd party cookies into the response. - # An example is "sticky session"/"insert cookie" persistence - # of a load balancer for a SHC. - header_list = list(response.info().items()) - header_list.append(("Set-Cookie", "BIGipServer_splunk-shc-8089=1234567890.12345.0000; path=/; Httponly; Secure")) - header_list.append(("Set-Cookie", "home_made=yummy")) - - return { - 'status': response.code, - 'reason': response.msg, - 'headers': header_list, - 'body': BytesIO(response.read()) - } - - -class TestCookiePersistence(testlib.SDKTestCase): - # Verify persistence of 3rd party inserted cookies. - def test_3rdPartyInsertedCookiePersistence(self): - paths = ["/services", "authentication/users", - "search/jobs"] - logging.debug("Connecting with urllib2_insert_cookie_handler %s", urllib2_insert_cookie_handler) - context = binding.connect( - handler=urllib2_insert_cookie_handler, - **self.opts.kwargs) - - persisted_cookies = context.get_cookies() - - splunk_token_found = False - for k, v in persisted_cookies.items(): - if k[:8] == "splunkd_": - splunk_token_found = True - break - - self.assertEqual(splunk_token_found, True) - self.assertEqual(persisted_cookies['BIGipServer_splunk-shc-8089'], "1234567890.12345.0000") - self.assertEqual(persisted_cookies['home_made'], "yummy") - - -@pytest.mark.smoke -class TestLogout(BindingTestCase): - def test_logout(self): - response = self.context.get("/services") - self.assertEqual(response.status, 200) - self.context.logout() - self.assertEqual(self.context.token, binding._NoAuthenticationToken) - self.assertEqual(self.context.get_cookies(), {}) - self.assertRaises(AuthenticationError, - self.context.get, "/services") - self.assertRaises(AuthenticationError, - self.context.post, "/services") - self.assertRaises(AuthenticationError, - self.context.delete, "/services") - self.context.login() - response = self.context.get("/services") - self.assertEqual(response.status, 200) - - -class TestCookieAuthentication(unittest.TestCase): - def setUp(self): - self.opts = testlib.parse([], {}, ".env") - self.context = binding.connect(**self.opts.kwargs) - - # Skip these tests if running below Splunk 6.2, cookie-auth didn't exist before - from splunklib import client - service = client.Service(**self.opts.kwargs) - # TODO: Workaround the fact that skipTest is not defined by unittest2.TestCase - service.login() - splver = service.splunk_version - if splver[:2] < (6, 2): - self.skipTest("Skipping cookie-auth tests, running in %d.%d.%d, this feature was added in 6.2+" % splver) - - if getattr(unittest.TestCase, 'assertIsNotNone', None) is None: - - def assertIsNotNone(self, obj, msg=None): - if obj is None: - raise self.failureException(msg or '%r is not None' % obj) - - @pytest.mark.smoke - def test_cookie_in_auth_headers(self): - self.assertIsNotNone(self.context._auth_headers) - self.assertNotEqual(self.context._auth_headers, []) - self.assertEqual(len(self.context._auth_headers), 1) - self.assertEqual(len(self.context._auth_headers), 1) - self.assertEqual(self.context._auth_headers[0][0], "Cookie") - self.assertEqual(self.context._auth_headers[0][1][:8], "splunkd_") - - @pytest.mark.smoke - def test_got_cookie_on_connect(self): - self.assertIsNotNone(self.context.get_cookies()) - self.assertNotEqual(self.context.get_cookies(), {}) - self.assertEqual(len(self.context.get_cookies()), 1) - self.assertEqual(list(self.context.get_cookies().keys())[0][:8], "splunkd_") - - @pytest.mark.smoke - def test_cookie_with_autologin(self): - self.context.autologin = True - self.assertEqual(self.context.get("/services").status, 200) - self.assertTrue(self.context.has_cookies()) - self.context.logout() - self.assertFalse(self.context.has_cookies()) - self.assertEqual(self.context.get("/services").status, 200) - self.assertTrue(self.context.has_cookies()) - - @pytest.mark.smoke - def test_cookie_without_autologin(self): - self.context.autologin = False - self.assertEqual(self.context.get("/services").status, 200) - self.assertTrue(self.context.has_cookies()) - self.context.logout() - self.assertFalse(self.context.has_cookies()) - self.assertRaises(AuthenticationError, - self.context.get, "/services") - - @pytest.mark.smoke - def test_got_updated_cookie_with_get(self): - old_cookies = self.context.get_cookies() - resp = self.context.get("apps/local") - found = False - for key, value in resp.headers: - if key.lower() == "set-cookie": - found = True - self.assertEqual(value[:8], "splunkd_") - - new_cookies = {} - binding._parse_cookies(value, new_cookies) - # We're only expecting 1 in this scenario - self.assertEqual(len(old_cookies), 1) - self.assertTrue(len(list(new_cookies.values())), 1) - self.assertEqual(old_cookies, new_cookies) - self.assertEqual(list(new_cookies.values())[0], list(old_cookies.values())[0]) - self.assertTrue(found) - - @pytest.mark.smoke - def test_login_fails_with_bad_cookie(self): - # We should get an error if using a bad cookie - try: - binding.connect(**{"cookie": "bad=cookie"}) - self.fail() - except AuthenticationError as ae: - self.assertEqual(str(ae), "Login failed.") - - @pytest.mark.smoke - def test_login_with_multiple_cookies(self): - # We should get an error if using a bad cookie - new_context = binding.Context() - new_context.get_cookies().update({"bad": "cookie"}) - try: - new_context = new_context.login() - self.fail() - except AuthenticationError as ae: - self.assertEqual(str(ae), "Login failed.") - # Bring in a valid cookie now - for key, value in self.context.get_cookies().items(): - new_context.get_cookies()[key] = value - - self.assertEqual(len(new_context.get_cookies()), 2) - self.assertTrue('bad' in list(new_context.get_cookies().keys())) - self.assertTrue('cookie' in list(new_context.get_cookies().values())) - - for k, v in self.context.get_cookies().items(): - self.assertEqual(new_context.get_cookies()[k], v) - - self.assertEqual(new_context.get("apps/local").status, 200) - - @pytest.mark.smoke - def test_login_fails_without_cookie_or_token(self): - opts = { - 'host': self.opts.kwargs['host'], - 'port': self.opts.kwargs['port'] - } - try: - binding.connect(**opts) - self.fail() - except AuthenticationError as ae: - self.assertEqual(str(ae), "Login failed.") - - -class TestNamespace(unittest.TestCase): - def test_namespace(self): - tests = [ - ({}, - {'sharing': None, 'owner': None, 'app': None}), - - ({'owner': "Bob"}, - {'sharing': None, 'owner': "Bob", 'app': None}), - - ({'app': "search"}, - {'sharing': None, 'owner': None, 'app': "search"}), - - ({'owner': "Bob", 'app': "search"}, - {'sharing': None, 'owner': "Bob", 'app': "search"}), - - ({'sharing': "user", 'owner': "Bob@bob.com"}, - {'sharing': "user", 'owner': "Bob@bob.com", 'app': None}), - - ({'sharing': "user"}, - {'sharing': "user", 'owner': None, 'app': None}), - - ({'sharing': "user", 'owner': "Bob"}, - {'sharing': "user", 'owner': "Bob", 'app': None}), - - ({'sharing': "user", 'app': "search"}, - {'sharing': "user", 'owner': None, 'app': "search"}), - - ({'sharing': "user", 'owner': "Bob", 'app': "search"}, - {'sharing': "user", 'owner': "Bob", 'app': "search"}), - - ({'sharing': "app"}, - {'sharing': "app", 'owner': "nobody", 'app': None}), - - ({'sharing': "app", 'owner': "Bob"}, - {'sharing': "app", 'owner': "nobody", 'app': None}), - - ({'sharing': "app", 'app': "search"}, - {'sharing': "app", 'owner': "nobody", 'app': "search"}), - - ({'sharing': "app", 'owner': "Bob", 'app': "search"}, - {'sharing': "app", 'owner': "nobody", 'app': "search"}), - - ({'sharing': "global"}, - {'sharing': "global", 'owner': "nobody", 'app': None}), - - ({'sharing': "global", 'owner': "Bob"}, - {'sharing': "global", 'owner': "nobody", 'app': None}), - - ({'sharing': "global", 'app': "search"}, - {'sharing': "global", 'owner': "nobody", 'app': "search"}), - - ({'sharing': "global", 'owner': "Bob", 'app': "search"}, - {'sharing': "global", 'owner': "nobody", 'app': "search"}), - - ({'sharing': "system"}, - {'sharing': "system", 'owner': "nobody", 'app': "system"}), - - ({'sharing': "system", 'owner': "Bob"}, - {'sharing': "system", 'owner': "nobody", 'app': "system"}), - - ({'sharing': "system", 'app': "search"}, - {'sharing': "system", 'owner': "nobody", 'app': "system"}), - - ({'sharing': "system", 'owner': "Bob", 'app': "search"}, - {'sharing': "system", 'owner': "nobody", 'app': "system"}), - - ({'sharing': 'user', 'owner': '-', 'app': '-'}, - {'sharing': 'user', 'owner': '-', 'app': '-'})] - - for kwargs, expected in tests: - namespace = binding.namespace(**kwargs) - for k, v in expected.items(): - self.assertEqual(namespace[k], v) - - def test_namespace_fails(self): - self.assertRaises(ValueError, binding.namespace, sharing="gobble") - - -@pytest.mark.smoke -class TestBasicAuthentication(unittest.TestCase): - def setUp(self): - self.opts = testlib.parse([], {}, ".env") - opts = self.opts.kwargs.copy() - opts["basic"] = True - opts["username"] = self.opts.kwargs["username"] - opts["password"] = self.opts.kwargs["password"] - - self.context = binding.connect(**opts) - from splunklib import client - service = client.Service(**opts) - - if getattr(unittest.TestCase, 'assertIsNotNone', None) is None: - def assertIsNotNone(self, obj, msg=None): - if obj is None: - raise self.failureException(msg or '%r is not None' % obj) - - def test_basic_in_auth_headers(self): - self.assertIsNotNone(self.context._auth_headers) - self.assertNotEqual(self.context._auth_headers, []) - self.assertEqual(len(self.context._auth_headers), 1) - self.assertEqual(len(self.context._auth_headers), 1) - self.assertEqual(self.context._auth_headers[0][0], "Authorization") - self.assertEqual(self.context._auth_headers[0][1][:6], "Basic ") - self.assertEqual(self.context.get("/services").status, 200) - - -@pytest.mark.smoke -class TestTokenAuthentication(BindingTestCase): - def test_preexisting_token(self): - token = self.context.token - opts = self.opts.kwargs.copy() - opts["token"] = token - opts["username"] = "boris the mad baboon" - opts["password"] = "nothing real" - - newContext = binding.Context(**opts) - response = newContext.get("/services") - self.assertEqual(response.status, 200) - - socket = newContext.connect() - socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) - socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) - socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) - socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) - socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) - socket.write("\r\n".encode('utf-8')) - socket.close() - - def test_preexisting_token_sans_splunk(self): - token = self.context.token - if token.startswith('Splunk '): - token = token.split(' ', 1)[1] - self.assertFalse(token.startswith('Splunk ')) - else: - self.fail('Token did not start with "Splunk ".') - opts = self.opts.kwargs.copy() - opts["token"] = token - opts["username"] = "boris the mad baboon" - opts["password"] = "nothing real" - - newContext = binding.Context(**opts) - response = newContext.get("/services") - self.assertEqual(response.status, 200) - - socket = newContext.connect() - socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) - socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) - socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) - socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) - socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) - socket.write("\r\n".encode('utf-8')) - socket.close() - - def test_connect_with_preexisting_token_sans_user_and_pass(self): - token = self.context.token - opts = self.opts.kwargs.copy() - del opts['username'] - del opts['password'] - opts["token"] = token - - newContext = binding.connect(**opts) - response = newContext.get('/services') - self.assertEqual(response.status, 200) - - socket = newContext.connect() - socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) - socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) - socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) - socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) - socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) - socket.write("\r\n".encode('utf-8')) - socket.close() - - -class TestPostWithBodyParam(unittest.TestCase): - - def test_post(self): - def handler(url, message, **kwargs): - assert url == "https://localhost:8089/servicesNS/testowner/testapp/foo/bar" - assert message["body"] == b"testkey=testvalue" - return splunklib.data.Record({ - "status": 200, - "headers": [], - }) - - ctx = binding.Context(handler=handler) - ctx.post("foo/bar", owner="testowner", app="testapp", body={"testkey": "testvalue"}) - - def test_post_with_params_and_body(self): - def handler(url, message, **kwargs): - assert url == "https://localhost:8089/servicesNS/testowner/testapp/foo/bar?extrakey=extraval" - assert message["body"] == b"testkey=testvalue" - return splunklib.data.Record({ - "status": 200, - "headers": [], - }) - - ctx = binding.Context(handler=handler) - ctx.post("foo/bar", extrakey="extraval", owner="testowner", app="testapp", body={"testkey": "testvalue"}) - - def test_post_with_params_and_no_body(self): - def handler(url, message, **kwargs): - assert url == "https://localhost:8089/servicesNS/testowner/testapp/foo/bar" - assert message["body"] == b"extrakey=extraval" - return splunklib.data.Record({ - "status": 200, - "headers": [], - }) - - ctx = binding.Context(handler=handler) - ctx.post("foo/bar", extrakey="extraval", owner="testowner", app="testapp") - - -def _wrap_handler(func, response_code=200, body=""): - def wrapped(handler_self): - result = func(handler_self) - if result is None: - handler_self.send_response(response_code) - handler_self.end_headers() - handler_self.wfile.write(body) - - return wrapped - - -class MockServer: - def __init__(self, port=9093, **handlers): - methods = {"do_" + k: _wrap_handler(v) for (k, v) in handlers.items()} - - def init(handler_self, socket, address, server): - BaseHTTPServer.BaseHTTPRequestHandler.__init__(handler_self, socket, address, server) - - def log(*args): # To silence server access logs - pass - - methods["__init__"] = init - methods["log_message"] = log - Handler = type("Handler", - (BaseHTTPServer.BaseHTTPRequestHandler, object), - methods) - self._svr = BaseHTTPServer.HTTPServer(("localhost", port), Handler) - - def run(): - self._svr.handle_request() - - self._thread = Thread(target=run) - self._thread.daemon = True - - def __enter__(self): - self._thread.start() - return self._svr - - def __exit__(self, typ, value, traceback): - self._thread.join(10) - self._svr.server_close() - - -class TestFullPost(unittest.TestCase): - - def test_post_with_body_urlencoded(self): - def check_response(handler): - length = int(handler.headers.get('content-length', 0)) - body = handler.rfile.read(length) - assert body.decode('utf-8') == "foo=bar" - - with MockServer(POST=check_response): - ctx = binding.connect(port=9093, scheme='http', token="waffle") - ctx.post("/", foo="bar") - - def test_post_with_body_string(self): - def check_response(handler): - length = int(handler.headers.get('content-length', 0)) - body = handler.rfile.read(length) - assert handler.headers['content-type'] == 'application/json' - assert json.loads(body)["baz"] == "baf" - - with MockServer(POST=check_response): - ctx = binding.connect(port=9093, scheme='http', token="waffle", - headers=[("Content-Type", "application/json")]) - ctx.post("/", foo="bar", body='{"baz": "baf"}') - - def test_post_with_body_dict(self): - def check_response(handler): - length = int(handler.headers.get('content-length', 0)) - body = handler.rfile.read(length) - assert handler.headers['content-type'] == 'application/x-www-form-urlencoded' - assert ensure_str(body) in ['baz=baf&hep=cat', 'hep=cat&baz=baf'] - - with MockServer(POST=check_response): - ctx = binding.connect(port=9093, scheme='http', token="waffle") - ctx.post("/", foo="bar", body={"baz": "baf", "hep": "cat"}) - - -if __name__ == "__main__": - unittest.main() +#!/usr/bin/env python +# +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from http import server as BaseHTTPServer +from io import BytesIO, StringIO +from threading import Thread +from urllib.request import Request, urlopen + +from xml.etree.ElementTree import XML + +import json +import logging +from tests import testlib +import unittest +import socket +import ssl + +import splunklib +from splunklib import binding +from splunklib.binding import HTTPError, AuthenticationError, UrlEncoded +from splunklib import data +from splunklib.utils import ensure_str + +import pytest + +# splunkd endpoint paths +PATH_USERS = "authentication/users/" + +# XML Namespaces +NAMESPACE_ATOM = "http://www.w3.org/2005/Atom" +NAMESPACE_REST = "http://dev.splunk.com/ns/rest" +NAMESPACE_OPENSEARCH = "http://a9.com/-/spec/opensearch/1.1" + +# XML Extended Name Fragments +XNAMEF_ATOM = "{%s}%%s" % NAMESPACE_ATOM +XNAMEF_REST = "{%s}%%s" % NAMESPACE_REST +XNAMEF_OPENSEARCH = "{%s}%%s" % NAMESPACE_OPENSEARCH + +# XML Extended Names +XNAME_AUTHOR = XNAMEF_ATOM % "author" +XNAME_ENTRY = XNAMEF_ATOM % "entry" +XNAME_FEED = XNAMEF_ATOM % "feed" +XNAME_ID = XNAMEF_ATOM % "id" +XNAME_TITLE = XNAMEF_ATOM % "title" + + +def load(response): + return data.load(response.body.read()) + + +class BindingTestCase(unittest.TestCase): + context = None + + def setUp(self): + logging.info("%s", self.__class__.__name__) + self.opts = testlib.parse([], {}, ".env") + self.context = binding.connect(**self.opts.kwargs) + logging.debug("Connected to splunkd.") + + +class TestResponseReader(BindingTestCase): + def test_empty(self): + response = binding.ResponseReader(BytesIO(b"")) + self.assertTrue(response.empty) + self.assertEqual(response.peek(10), b"") + self.assertEqual(response.read(10), b"") + + arr = bytearray(10) + self.assertEqual(response.readinto(arr), 0) + self.assertEqual(arr, bytearray(10)) + self.assertTrue(response.empty) + + def test_read_past_end(self): + txt = b"abcd" + response = binding.ResponseReader(BytesIO(txt)) + self.assertFalse(response.empty) + self.assertEqual(response.peek(10), txt) + self.assertEqual(response.read(10), txt) + self.assertTrue(response.empty) + self.assertEqual(response.peek(10), b"") + self.assertEqual(response.read(10), b"") + + def test_read_partial(self): + txt = b"This is a test of the emergency broadcasting system." + response = binding.ResponseReader(BytesIO(txt)) + self.assertEqual(response.peek(5), txt[:5]) + self.assertFalse(response.empty) + self.assertEqual(response.read(), txt) + self.assertTrue(response.empty) + self.assertEqual(response.read(), b'') + + def test_readable(self): + txt = "abcd" + response = binding.ResponseReader(StringIO(txt)) + self.assertTrue(response.readable()) + + def test_readinto_bytearray(self): + txt = b"Checking readinto works as expected" + response = binding.ResponseReader(BytesIO(txt)) + arr = bytearray(10) + self.assertEqual(response.readinto(arr), 10) + self.assertEqual(arr[:10], b"Checking r") + self.assertEqual(response.readinto(arr), 10) + self.assertEqual(arr[:10], b"eadinto wo") + self.assertEqual(response.readinto(arr), 10) + self.assertEqual(arr[:10], b"rks as exp") + self.assertEqual(response.readinto(arr), 5) + self.assertEqual(arr[:5], b"ected") + self.assertTrue(response.empty) + + def test_readinto_memoryview(self): + txt = b"Checking readinto works as expected" + response = binding.ResponseReader(BytesIO(txt)) + arr = bytearray(10) + mv = memoryview(arr) + self.assertEqual(response.readinto(mv), 10) + self.assertEqual(arr[:10], b"Checking r") + self.assertEqual(response.readinto(mv), 10) + self.assertEqual(arr[:10], b"eadinto wo") + self.assertEqual(response.readinto(mv), 10) + self.assertEqual(arr[:10], b"rks as exp") + self.assertEqual(response.readinto(mv), 5) + self.assertEqual(arr[:5], b"ected") + self.assertTrue(response.empty) + + +class TestUrlEncoded(BindingTestCase): + def test_idempotent(self): + a = UrlEncoded('abc') + self.assertEqual(a, UrlEncoded(a)) + + def test_append(self): + self.assertEqual(UrlEncoded('a') + UrlEncoded('b'), + UrlEncoded('ab')) + + def test_append_string(self): + self.assertEqual(UrlEncoded('a') + '%', + UrlEncoded('a%')) + + def test_append_to_string(self): + self.assertEqual('%' + UrlEncoded('a'), + UrlEncoded('%a')) + + def test_interpolation_fails(self): + self.assertRaises(TypeError, lambda: UrlEncoded('%s') % 'boris') + + def test_chars(self): + for char, code in [(' ', '%20'), + ('"', '%22'), + ('%', '%25')]: + self.assertEqual(UrlEncoded(char), + UrlEncoded(code, skip_encode=True)) + + def test_repr(self): + self.assertEqual(repr(UrlEncoded('% %')), "UrlEncoded('% %')") + + +class TestAuthority(unittest.TestCase): + def test_authority_default(self): + self.assertEqual(binding._authority(), + "https://localhost:8089") + + def test_ipv4_host(self): + self.assertEqual( + binding._authority( + host="splunk.utopia.net"), + "https://splunk.utopia.net:8089") + + def test_ipv6_host(self): + self.assertEqual( + binding._authority( + host="2001:0db8:85a3:0000:0000:8a2e:0370:7334"), + "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089") + + def test_ipv6_host_enclosed(self): + self.assertEqual( + binding._authority( + host="[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"), + "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:8089") + + def test_all_fields(self): + self.assertEqual( + binding._authority( + scheme="http", + host="splunk.utopia.net", + port="471"), + "http://splunk.utopia.net:471") + + +class TestUserManipulation(BindingTestCase): + def setUp(self): + BindingTestCase.setUp(self) + self.username = testlib.tmpname() + self.password = "changeme!" + self.roles = "power" + + # Delete user if it exists already + try: + response = self.context.delete(PATH_USERS + self.username) + self.assertEqual(response.status, 200) + except HTTPError as e: + self.assertTrue(e.status in [400, 500]) + + def tearDown(self): + BindingTestCase.tearDown(self) + try: + self.context.delete(PATH_USERS + self.username) + except HTTPError as e: + if e.status not in [400, 500]: + raise + + def test_user_without_role_fails(self): + self.assertRaises(binding.HTTPError, + self.context.post, + PATH_USERS, name=self.username, + password=self.password) + + def test_create_user(self): + response = self.context.post( + PATH_USERS, name=self.username, + password=self.password, roles=self.roles) + self.assertEqual(response.status, 201) + + response = self.context.get(PATH_USERS + self.username) + entry = load(response).feed.entry + self.assertEqual(entry.title, self.username) + + def test_update_user(self): + self.test_create_user() + response = self.context.post( + PATH_USERS + self.username, + password=self.password, + roles=self.roles, + defaultApp="search", + realname="Renzo", + email="email.me@now.com") + self.assertEqual(response.status, 200) + + response = self.context.get(PATH_USERS + self.username) + self.assertEqual(response.status, 200) + entry = load(response).feed.entry + self.assertEqual(entry.title, self.username) + self.assertEqual(entry.content.defaultApp, "search") + self.assertEqual(entry.content.realname, "Renzo") + self.assertEqual(entry.content.email, "email.me@now.com") + + def test_post_with_body_behaves(self): + self.test_create_user() + response = self.context.post( + PATH_USERS + self.username, + body="defaultApp=search", + ) + self.assertEqual(response.status, 200) + + def test_post_with_get_arguments_to_receivers_stream(self): + text = 'Hello, world!' + response = self.context.post( + '/services/receivers/simple', + headers=[('x-splunk-input-mode', 'streaming')], + source='sdk', sourcetype='sdk_test', + body=text + ) + self.assertEqual(response.status, 200) + + +class TestSocket(BindingTestCase): + def test_socket(self): + socket = self.context.connect() + socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) + socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) + socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) + socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) + socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) + socket.write("\r\n".encode('utf-8')) + socket.close() + + # Sockets take bytes not strings + # + # def test_unicode_socket(self): + # socket = self.context.connect() + # socket.write(u"POST %s HTTP/1.1\r\n" %\ + # self.context._abspath("some/path/to/post/to")) + # socket.write(u"Host: %s:%s\r\n" %\ + # (self.context.host, self.context.port)) + # socket.write(u"Accept-Encoding: identity\r\n") + # socket.write((u"Authorization: %s\r\n" %\ + # self.context.token).encode('utf-8')) + # socket.write(u"X-Splunk-Input-Mode: Streaming\r\n") + # socket.write("\r\n") + # socket.close() + + def test_socket_gethostbyname(self): + self.assertTrue(self.context.connect()) + self.context.host = socket.gethostbyname(self.context.host) + self.assertTrue(self.context.connect()) + + +class TestUnicodeConnect(BindingTestCase): + def test_unicode_connect(self): + opts = self.opts.kwargs.copy() + opts['host'] = str(opts['host']) + context = binding.connect(**opts) + # Just check to make sure the service is alive + response = context.get("/services") + self.assertEqual(response.status, 200) + + +@pytest.mark.smoke +class TestAutologin(BindingTestCase): + def test_with_autologin(self): + self.context.autologin = True + self.assertEqual(self.context.get("/services").status, 200) + self.context.logout() + self.assertEqual(self.context.get("/services").status, 200) + + def test_without_autologin(self): + self.context.autologin = False + self.assertEqual(self.context.get("/services").status, 200) + self.context.logout() + self.assertRaises(AuthenticationError, + self.context.get, "/services") + + +class TestAbspath(BindingTestCase): + def setUp(self): + BindingTestCase.setUp(self) + self.kwargs = self.opts.kwargs.copy() + if 'app' in self.kwargs: del self.kwargs['app'] + if 'owner' in self.kwargs: del self.kwargs['owner'] + + def test_default(self): + path = self.context._abspath("foo", owner=None, app=None) + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/services/foo") + + def test_with_owner(self): + path = self.context._abspath("foo", owner="me", app=None) + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me/system/foo") + + def test_with_app(self): + path = self.context._abspath("foo", owner=None, app="MyApp") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") + + def test_with_both(self): + path = self.context._abspath("foo", owner="me", app="MyApp") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me/MyApp/foo") + + def test_user_sharing(self): + path = self.context._abspath("foo", owner="me", app="MyApp", sharing="user") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me/MyApp/foo") + + def test_sharing_app(self): + path = self.context._abspath("foo", owner="me", app="MyApp", sharing="app") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") + + def test_sharing_global(self): + path = self.context._abspath("foo", owner="me", app="MyApp", sharing="global") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") + + def test_sharing_system(self): + path = self.context._abspath("foo bar", owner="me", app="MyApp", sharing="system") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/system/foo%20bar") + + def test_url_forbidden_characters(self): + path = self.context._abspath('/a/b c/d') + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, '/a/b%20c/d') + + def test_context_defaults(self): + context = binding.connect(**self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/services/foo") + + def test_context_with_owner(self): + context = binding.connect(owner="me", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me/system/foo") + + def test_context_with_app(self): + context = binding.connect(app="MyApp", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") + + def test_context_with_both(self): + context = binding.connect(owner="me", app="MyApp", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me/MyApp/foo") + + def test_context_with_user_sharing(self): + context = binding.connect( + owner="me", app="MyApp", sharing="user", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me/MyApp/foo") + + def test_context_with_app_sharing(self): + context = binding.connect( + owner="me", app="MyApp", sharing="app", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") + + def test_context_with_global_sharing(self): + context = binding.connect( + owner="me", app="MyApp", sharing="global", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/MyApp/foo") + + def test_context_with_system_sharing(self): + context = binding.connect( + owner="me", app="MyApp", sharing="system", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/nobody/system/foo") + + def test_context_with_owner_as_email(self): + context = binding.connect(owner="me@me.com", **self.kwargs) + path = context._abspath("foo") + self.assertTrue(isinstance(path, UrlEncoded)) + self.assertEqual(path, "/servicesNS/me%40me.com/system/foo") + self.assertEqual(path, UrlEncoded("/servicesNS/me@me.com/system/foo")) + + +# An urllib2 based HTTP request handler, used to test the binding layers +# support for pluggable request handlers. +def urllib2_handler(url, message, **kwargs): + method = message['method'].lower() + data = message.get('body', b"") if method == 'post' else None + headers = dict(message.get('headers', [])) + req = Request(url, data, headers) + try: + response = urlopen(req, context=ssl._create_unverified_context()) + except HTTPError as response: + pass # Propagate HTTP errors via the returned response message + return { + 'status': response.code, + 'reason': response.msg, + 'headers': dict(response.info()), + 'body': BytesIO(response.read()) + } + + +def isatom(body): + """Answers if the given response body looks like ATOM.""" + root = XML(body) + return \ + root.tag == XNAME_FEED and \ + root.find(XNAME_AUTHOR) is not None and \ + root.find(XNAME_ID) is not None and \ + root.find(XNAME_TITLE) is not None + + +class TestPluggableHTTP(testlib.SDKTestCase): + # Verify pluggable HTTP reqeust handlers. + def test_handlers(self): + paths = ["/services", "authentication/users", + "search/jobs"] + handlers = [binding.handler(), # default handler + urllib2_handler] + for handler in handlers: + logging.debug("Connecting with handler %s", handler) + context = binding.connect( + handler=handler, + **self.opts.kwargs) + for path in paths: + body = context.get(path).body.read() + self.assertTrue(isatom(body)) + + +def urllib2_insert_cookie_handler(url, message, **kwargs): + method = message['method'].lower() + data = message.get('body', b"") if method == 'post' else None + headers = dict(message.get('headers', [])) + req = Request(url, data, headers) + try: + response = urlopen(req, context=ssl._create_unverified_context()) + except HTTPError as response: + pass # Propagate HTTP errors via the returned response message + + # Mimic the insertion of 3rd party cookies into the response. + # An example is "sticky session"/"insert cookie" persistence + # of a load balancer for a SHC. + header_list = list(response.info().items()) + header_list.append(("Set-Cookie", "BIGipServer_splunk-shc-8089=1234567890.12345.0000; path=/; Httponly; Secure")) + header_list.append(("Set-Cookie", "home_made=yummy")) + + return { + 'status': response.code, + 'reason': response.msg, + 'headers': header_list, + 'body': BytesIO(response.read()) + } + + +class TestCookiePersistence(testlib.SDKTestCase): + # Verify persistence of 3rd party inserted cookies. + def test_3rdPartyInsertedCookiePersistence(self): + paths = ["/services", "authentication/users", + "search/jobs"] + logging.debug("Connecting with urllib2_insert_cookie_handler %s", urllib2_insert_cookie_handler) + context = binding.connect( + handler=urllib2_insert_cookie_handler, + **self.opts.kwargs) + + persisted_cookies = context.get_cookies() + + splunk_token_found = False + for k, v in persisted_cookies.items(): + if k[:8] == "splunkd_": + splunk_token_found = True + break + + self.assertEqual(splunk_token_found, True) + self.assertEqual(persisted_cookies['BIGipServer_splunk-shc-8089'], "1234567890.12345.0000") + self.assertEqual(persisted_cookies['home_made'], "yummy") + + +@pytest.mark.smoke +class TestLogout(BindingTestCase): + def test_logout(self): + response = self.context.get("/services") + self.assertEqual(response.status, 200) + self.context.logout() + self.assertEqual(self.context.token, binding._NoAuthenticationToken) + self.assertEqual(self.context.get_cookies(), {}) + self.assertRaises(AuthenticationError, + self.context.get, "/services") + self.assertRaises(AuthenticationError, + self.context.post, "/services") + self.assertRaises(AuthenticationError, + self.context.delete, "/services") + self.context.login() + response = self.context.get("/services") + self.assertEqual(response.status, 200) + + +class TestCookieAuthentication(unittest.TestCase): + def setUp(self): + self.opts = testlib.parse([], {}, ".env") + self.context = binding.connect(**self.opts.kwargs) + + # Skip these tests if running below Splunk 6.2, cookie-auth didn't exist before + from splunklib import client + service = client.Service(**self.opts.kwargs) + # TODO: Workaround the fact that skipTest is not defined by unittest2.TestCase + service.login() + splver = service.splunk_version + if splver[:2] < (6, 2): + self.skipTest("Skipping cookie-auth tests, running in %d.%d.%d, this feature was added in 6.2+" % splver) + + if getattr(unittest.TestCase, 'assertIsNotNone', None) is None: + + def assertIsNotNone(self, obj, msg=None): + if obj is None: + raise self.failureException(msg or '%r is not None' % obj) + + @pytest.mark.smoke + def test_cookie_in_auth_headers(self): + self.assertIsNotNone(self.context._auth_headers) + self.assertNotEqual(self.context._auth_headers, []) + self.assertEqual(len(self.context._auth_headers), 1) + self.assertEqual(len(self.context._auth_headers), 1) + self.assertEqual(self.context._auth_headers[0][0], "Cookie") + self.assertEqual(self.context._auth_headers[0][1][:8], "splunkd_") + + @pytest.mark.smoke + def test_got_cookie_on_connect(self): + self.assertIsNotNone(self.context.get_cookies()) + self.assertNotEqual(self.context.get_cookies(), {}) + self.assertEqual(len(self.context.get_cookies()), 1) + self.assertEqual(list(self.context.get_cookies().keys())[0][:8], "splunkd_") + + @pytest.mark.smoke + def test_cookie_with_autologin(self): + self.context.autologin = True + self.assertEqual(self.context.get("/services").status, 200) + self.assertTrue(self.context.has_cookies()) + self.context.logout() + self.assertFalse(self.context.has_cookies()) + self.assertEqual(self.context.get("/services").status, 200) + self.assertTrue(self.context.has_cookies()) + + @pytest.mark.smoke + def test_cookie_without_autologin(self): + self.context.autologin = False + self.assertEqual(self.context.get("/services").status, 200) + self.assertTrue(self.context.has_cookies()) + self.context.logout() + self.assertFalse(self.context.has_cookies()) + self.assertRaises(AuthenticationError, + self.context.get, "/services") + + @pytest.mark.smoke + def test_got_updated_cookie_with_get(self): + old_cookies = self.context.get_cookies() + resp = self.context.get("apps/local") + found = False + for key, value in resp.headers: + if key.lower() == "set-cookie": + found = True + self.assertEqual(value[:8], "splunkd_") + + new_cookies = {} + binding._parse_cookies(value, new_cookies) + # We're only expecting 1 in this scenario + self.assertEqual(len(old_cookies), 1) + self.assertTrue(len(list(new_cookies.values())), 1) + self.assertEqual(old_cookies, new_cookies) + self.assertEqual(list(new_cookies.values())[0], list(old_cookies.values())[0]) + self.assertTrue(found) + + @pytest.mark.smoke + def test_login_fails_with_bad_cookie(self): + # We should get an error if using a bad cookie + try: + binding.connect(**{"cookie": "bad=cookie"}) + self.fail() + except AuthenticationError as ae: + self.assertEqual(str(ae), "Login failed.") + + @pytest.mark.smoke + def test_login_with_multiple_cookies(self): + # We should get an error if using a bad cookie + new_context = binding.Context() + new_context.get_cookies().update({"bad": "cookie"}) + try: + new_context = new_context.login() + self.fail() + except AuthenticationError as ae: + self.assertEqual(str(ae), "Login failed.") + # Bring in a valid cookie now + for key, value in self.context.get_cookies().items(): + new_context.get_cookies()[key] = value + + self.assertEqual(len(new_context.get_cookies()), 2) + self.assertTrue('bad' in list(new_context.get_cookies().keys())) + self.assertTrue('cookie' in list(new_context.get_cookies().values())) + + for k, v in self.context.get_cookies().items(): + self.assertEqual(new_context.get_cookies()[k], v) + + self.assertEqual(new_context.get("apps/local").status, 200) + + @pytest.mark.smoke + def test_login_fails_without_cookie_or_token(self): + opts = { + 'host': self.opts.kwargs['host'], + 'port': self.opts.kwargs['port'] + } + try: + binding.connect(**opts) + self.fail() + except AuthenticationError as ae: + self.assertEqual(str(ae), "Login failed.") + + +class TestNamespace(unittest.TestCase): + def test_namespace(self): + tests = [ + ({}, + {'sharing': None, 'owner': None, 'app': None}), + + ({'owner': "Bob"}, + {'sharing': None, 'owner': "Bob", 'app': None}), + + ({'app': "search"}, + {'sharing': None, 'owner': None, 'app': "search"}), + + ({'owner': "Bob", 'app': "search"}, + {'sharing': None, 'owner': "Bob", 'app': "search"}), + + ({'sharing': "user", 'owner': "Bob@bob.com"}, + {'sharing': "user", 'owner': "Bob@bob.com", 'app': None}), + + ({'sharing': "user"}, + {'sharing': "user", 'owner': None, 'app': None}), + + ({'sharing': "user", 'owner': "Bob"}, + {'sharing': "user", 'owner': "Bob", 'app': None}), + + ({'sharing': "user", 'app': "search"}, + {'sharing': "user", 'owner': None, 'app': "search"}), + + ({'sharing': "user", 'owner': "Bob", 'app': "search"}, + {'sharing': "user", 'owner': "Bob", 'app': "search"}), + + ({'sharing': "app"}, + {'sharing': "app", 'owner': "nobody", 'app': None}), + + ({'sharing': "app", 'owner': "Bob"}, + {'sharing': "app", 'owner': "nobody", 'app': None}), + + ({'sharing': "app", 'app': "search"}, + {'sharing': "app", 'owner': "nobody", 'app': "search"}), + + ({'sharing': "app", 'owner': "Bob", 'app': "search"}, + {'sharing': "app", 'owner': "nobody", 'app': "search"}), + + ({'sharing': "global"}, + {'sharing': "global", 'owner': "nobody", 'app': None}), + + ({'sharing': "global", 'owner': "Bob"}, + {'sharing': "global", 'owner': "nobody", 'app': None}), + + ({'sharing': "global", 'app': "search"}, + {'sharing': "global", 'owner': "nobody", 'app': "search"}), + + ({'sharing': "global", 'owner': "Bob", 'app': "search"}, + {'sharing': "global", 'owner': "nobody", 'app': "search"}), + + ({'sharing': "system"}, + {'sharing': "system", 'owner': "nobody", 'app': "system"}), + + ({'sharing': "system", 'owner': "Bob"}, + {'sharing': "system", 'owner': "nobody", 'app': "system"}), + + ({'sharing': "system", 'app': "search"}, + {'sharing': "system", 'owner': "nobody", 'app': "system"}), + + ({'sharing': "system", 'owner': "Bob", 'app': "search"}, + {'sharing': "system", 'owner': "nobody", 'app': "system"}), + + ({'sharing': 'user', 'owner': '-', 'app': '-'}, + {'sharing': 'user', 'owner': '-', 'app': '-'})] + + for kwargs, expected in tests: + namespace = binding.namespace(**kwargs) + for k, v in expected.items(): + self.assertEqual(namespace[k], v) + + def test_namespace_fails(self): + self.assertRaises(ValueError, binding.namespace, sharing="gobble") + + +@pytest.mark.smoke +class TestBasicAuthentication(unittest.TestCase): + def setUp(self): + self.opts = testlib.parse([], {}, ".env") + opts = self.opts.kwargs.copy() + opts["basic"] = True + opts["username"] = self.opts.kwargs["username"] + opts["password"] = self.opts.kwargs["password"] + + self.context = binding.connect(**opts) + from splunklib import client + service = client.Service(**opts) + + if getattr(unittest.TestCase, 'assertIsNotNone', None) is None: + def assertIsNotNone(self, obj, msg=None): + if obj is None: + raise self.failureException(msg or '%r is not None' % obj) + + def test_basic_in_auth_headers(self): + self.assertIsNotNone(self.context._auth_headers) + self.assertNotEqual(self.context._auth_headers, []) + self.assertEqual(len(self.context._auth_headers), 1) + self.assertEqual(len(self.context._auth_headers), 1) + self.assertEqual(self.context._auth_headers[0][0], "Authorization") + self.assertEqual(self.context._auth_headers[0][1][:6], "Basic ") + self.assertEqual(self.context.get("/services").status, 200) + + +@pytest.mark.smoke +class TestTokenAuthentication(BindingTestCase): + def test_preexisting_token(self): + token = self.context.token + opts = self.opts.kwargs.copy() + opts["token"] = token + opts["username"] = "boris the mad baboon" + opts["password"] = "nothing real" + + newContext = binding.Context(**opts) + response = newContext.get("/services") + self.assertEqual(response.status, 200) + + socket = newContext.connect() + socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) + socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) + socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) + socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) + socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) + socket.write("\r\n".encode('utf-8')) + socket.close() + + def test_preexisting_token_sans_splunk(self): + token = self.context.token + if token.startswith('Splunk '): + token = token.split(' ', 1)[1] + self.assertFalse(token.startswith('Splunk ')) + else: + self.fail('Token did not start with "Splunk ".') + opts = self.opts.kwargs.copy() + opts["token"] = token + opts["username"] = "boris the mad baboon" + opts["password"] = "nothing real" + + newContext = binding.Context(**opts) + response = newContext.get("/services") + self.assertEqual(response.status, 200) + + socket = newContext.connect() + socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) + socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) + socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) + socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) + socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) + socket.write("\r\n".encode('utf-8')) + socket.close() + + def test_connect_with_preexisting_token_sans_user_and_pass(self): + token = self.context.token + opts = self.opts.kwargs.copy() + del opts['username'] + del opts['password'] + opts["token"] = token + + newContext = binding.connect(**opts) + response = newContext.get('/services') + self.assertEqual(response.status, 200) + + socket = newContext.connect() + socket.write((f"POST {self.context._abspath('some/path/to/post/to')} HTTP/1.1\r\n").encode('utf-8')) + socket.write((f"Host: {self.context.host}:{self.context.port}\r\n").encode('utf-8')) + socket.write("Accept-Encoding: identity\r\n".encode('utf-8')) + socket.write((f"Authorization: {self.context.token}\r\n").encode('utf-8')) + socket.write("X-Splunk-Input-Mode: Streaming\r\n".encode('utf-8')) + socket.write("\r\n".encode('utf-8')) + socket.close() + + +class TestPostWithBodyParam(unittest.TestCase): + + def test_post(self): + def handler(url, message, **kwargs): + assert url == "https://localhost:8089/servicesNS/testowner/testapp/foo/bar" + assert message["body"] == b"testkey=testvalue" + return splunklib.data.Record({ + "status": 200, + "headers": [], + }) + + ctx = binding.Context(handler=handler) + ctx.post("foo/bar", owner="testowner", app="testapp", body={"testkey": "testvalue"}) + + def test_post_with_params_and_body(self): + def handler(url, message, **kwargs): + assert url == "https://localhost:8089/servicesNS/testowner/testapp/foo/bar?extrakey=extraval" + assert message["body"] == b"testkey=testvalue" + return splunklib.data.Record({ + "status": 200, + "headers": [], + }) + + ctx = binding.Context(handler=handler) + ctx.post("foo/bar", extrakey="extraval", owner="testowner", app="testapp", body={"testkey": "testvalue"}) + + def test_post_with_params_and_no_body(self): + def handler(url, message, **kwargs): + assert url == "https://localhost:8089/servicesNS/testowner/testapp/foo/bar" + assert message["body"] == b"extrakey=extraval" + return splunklib.data.Record({ + "status": 200, + "headers": [], + }) + + ctx = binding.Context(handler=handler) + ctx.post("foo/bar", extrakey="extraval", owner="testowner", app="testapp") + + +def _wrap_handler(func, response_code=200, body=""): + def wrapped(handler_self): + result = func(handler_self) + if result is None: + handler_self.send_response(response_code) + handler_self.end_headers() + handler_self.wfile.write(body) + + return wrapped + + +class MockServer: + def __init__(self, port=9093, **handlers): + methods = {"do_" + k: _wrap_handler(v) for (k, v) in handlers.items()} + + def init(handler_self, socket, address, server): + BaseHTTPServer.BaseHTTPRequestHandler.__init__(handler_self, socket, address, server) + + def log(*args): # To silence server access logs + pass + + methods["__init__"] = init + methods["log_message"] = log + Handler = type("Handler", + (BaseHTTPServer.BaseHTTPRequestHandler, object), + methods) + self._svr = BaseHTTPServer.HTTPServer(("localhost", port), Handler) + + def run(): + self._svr.handle_request() + + self._thread = Thread(target=run) + self._thread.daemon = True + + def __enter__(self): + self._thread.start() + return self._svr + + def __exit__(self, typ, value, traceback): + self._thread.join(10) + self._svr.server_close() + + +class TestFullPost(unittest.TestCase): + + def test_post_with_body_urlencoded(self): + def check_response(handler): + length = int(handler.headers.get('content-length', 0)) + body = handler.rfile.read(length) + assert body.decode('utf-8') == "foo=bar" + + with MockServer(POST=check_response): + ctx = binding.connect(port=9093, scheme='http', token="waffle") + ctx.post("/", foo="bar") + + def test_post_with_body_string(self): + def check_response(handler): + length = int(handler.headers.get('content-length', 0)) + body = handler.rfile.read(length) + assert handler.headers['content-type'] == 'application/json' + assert json.loads(body)["baz"] == "baf" + + with MockServer(POST=check_response): + ctx = binding.connect(port=9093, scheme='http', token="waffle", + headers=[("Content-Type", "application/json")]) + ctx.post("/", foo="bar", body='{"baz": "baf"}') + + def test_post_with_body_dict(self): + def check_response(handler): + length = int(handler.headers.get('content-length', 0)) + body = handler.rfile.read(length) + assert handler.headers['content-type'] == 'application/x-www-form-urlencoded' + assert ensure_str(body) in ['baz=baf&hep=cat', 'hep=cat&baz=baf'] + + with MockServer(POST=check_response): + ctx = binding.connect(port=9093, scheme='http', token="waffle") + ctx.post("/", foo="bar", body={"baz": "baf", "hep": "cat"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/testlib.py b/tests/testlib.py index a92790e2..e7c7b6a7 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -1,261 +1,261 @@ -#!/usr/bin/env python -# -# Copyright © 2011-2024 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Shared unit test utilities.""" -import contextlib - -import os -import time -import logging -import sys - -# Run the test suite on the SDK without installing it. -sys.path.insert(0, '../') - -from time import sleep -from datetime import datetime, timedelta - -import unittest - -from utils import parse - -from splunklib import client - - - -logging.basicConfig( - filename='test.log', - level=logging.DEBUG, - format="%(asctime)s:%(levelname)s:%(message)s") - - -class NoRestartRequiredError(Exception): - pass - - -class WaitTimedOutError(Exception): - pass - - -def to_bool(x): - if x == '1': - return True - if x == '0': - return False - raise ValueError(f"Not a boolean value: {x}") - - -def tmpname(): - name = 'delete-me-' + str(os.getpid()) + str(time.time()).replace('.', '-') - return name - - -def wait(predicate, timeout=60, pause_time=0.5): - assert pause_time < timeout - start = datetime.now() - diff = timedelta(seconds=timeout) - while not predicate(): - if datetime.now() - start > diff: - logging.debug("wait timed out after %d seconds", timeout) - raise WaitTimedOutError - sleep(pause_time) - logging.debug("wait finished after %s seconds", datetime.now() - start) - - -class SDKTestCase(unittest.TestCase): - restart_already_required = False - installedApps = [] - - def assertEventuallyTrue(self, predicate, timeout=30, pause_time=0.5, - timeout_message="Operation timed out."): - assert pause_time < timeout - start = datetime.now() - diff = timedelta(seconds=timeout) - while not predicate(): - if datetime.now() - start > diff: - logging.debug("wait timed out after %d seconds", timeout) - self.fail(timeout_message) - sleep(pause_time) - logging.debug("wait finished after %s seconds", datetime.now() - start) - - def check_content(self, entity, **kwargs): - for k, v in kwargs: - self.assertEqual(entity[k], str(v)) - - def check_entity(self, entity): - assert entity is not None - self.assertTrue(entity.name is not None) - self.assertTrue(entity.path is not None) - - self.assertTrue(entity.state is not None) - self.assertTrue(entity.content is not None) - - # Verify access metadata - assert entity.access is not None - entity.access.app - entity.access.owner - entity.access.sharing - - # Verify content metadata - - # In some cases, the REST API does not return field metadata for when - # entities are intially listed by a collection, so we refresh to make - # sure the metadata is available. - entity.refresh() - - self.assertTrue(isinstance(entity.fields.required, list)) - self.assertTrue(isinstance(entity.fields.optional, list)) - self.assertTrue(isinstance(entity.fields.wildcard, list)) - - # Verify that all required fields appear in entity content - - for field in entity.fields.required: - try: - self.assertTrue(field in entity.content) - except: - # Check for known exceptions - if "configs/conf-times" in entity.path: - if field in ["is_sub_menu"]: - continue - raise - - def clear_restart_message(self): - """Tell Splunk to forget that it needs to be restarted. - - This is used mostly in cases such as deleting a temporary application. - Splunk asks to be restarted when that happens, but unless the application - contained modular input kinds or the like, it isn't necessary. - """ - if not self.service.restart_required: - raise ValueError("Tried to clear restart message when there was none.") - try: - self.service.delete("messages/restart_required") - except client.HTTPError as he: - if he.status != 404: - raise - - @contextlib.contextmanager - def fake_splunk_version(self, version): - original_version = self.service.splunk_version - try: - self.service._splunk_version = version - yield - finally: - self.service._splunk_version = original_version - - def install_app_from_collection(self, name): - collectionName = 'sdkappcollection' - if collectionName not in self.service.apps: - raise ValueError("sdk-test-application not installed in splunkd") - appPath = self.pathInApp(collectionName, ["build", name + ".tar"]) - kwargs = {"update": True, "name": appPath, "filename": True} - - try: - self.service.post("apps/local", **kwargs) - except client.HTTPError as he: - if he.status == 400: - raise IOError(f"App {name} not found in app collection") - if self.service.restart_required: - self.service.restart(120) - self.installedApps.append(name) - - def app_collection_installed(self): - collectionName = 'sdkappcollection' - return collectionName in self.service.apps - - def pathInApp(self, appName, pathComponents): - r"""Return a path to *pathComponents* in *appName*. - - `pathInApp` is used to refer to files in applications installed with - `install_app_from_collection`. For example, the app `file_to_upload` in - the collection contains `log.txt`. To get the path to it, call:: - - pathInApp('file_to_upload', ['log.txt']) - - The path to `setup.xml` in `has_setup_xml` would be fetched with:: - - pathInApp('has_setup_xml', ['default', 'setup.xml']) - - `pathInApp` figures out the correct separator to use (based on whether - splunkd is running on Windows or Unix) and joins the elements in - *pathComponents* into a path relative to the application specified by - *appName*. - - *pathComponents* should be a list of strings giving the components. - This function will try to figure out the correct separator (/ or \) - for the platform that splunkd is running on and construct the path - as needed. - - :return: A string giving the path. - """ - splunkHome = self.service.settings['SPLUNK_HOME'] - if "\\" in splunkHome: - # This clause must come first, since Windows machines may - # have mixed \ and / in their paths. - separator = "\\" - elif "/" in splunkHome: - separator = "/" - else: - raise ValueError("No separators in $SPLUNK_HOME. Can't determine what file separator to use.") - appPath = separator.join([splunkHome, "etc", "apps", appName] + pathComponents) - return appPath - - def uncheckedRestartSplunk(self, timeout=240): - self.service.restart(timeout) - - def restartSplunk(self, timeout=240): - if self.service.restart_required: - self.service.restart(timeout) - else: - raise NoRestartRequiredError() - - @classmethod - def setUpClass(cls): - cls.opts = parse([], {}, ".env") - cls.opts.kwargs.update({'retries': 3}) - # Before we start, make sure splunk doesn't need a restart. - service = client.connect(**cls.opts.kwargs) - if service.restart_required: - service.restart(timeout=120) - - def setUp(self): - unittest.TestCase.setUp(self) - self.opts.kwargs.update({'retries': 3}) - self.service = client.connect(**self.opts.kwargs) - # If Splunk is in a state requiring restart, go ahead - # and restart. That way we'll be sane for the rest of - # the test. - if self.service.restart_required: - self.restartSplunk() - logging.debug("Connected to splunkd version %s", '.'.join(str(x) for x in self.service.splunk_version)) - - def tearDown(self): - from splunklib.binding import HTTPError - - if self.service.restart_required: - self.fail("Test left Splunk in a state requiring a restart.") - - for appName in self.installedApps: - if appName in self.service.apps: - try: - self.service.apps.delete(appName) - wait(lambda: appName not in self.service.apps) - except HTTPError as error: - if not (os.name == 'nt' and error.status == 500): - raise - print(f'Ignoring failure to delete {appName} during tear down: {error}') - if self.service.restart_required: - self.clear_restart_message() +#!/usr/bin/env python +# +# Copyright © 2011-2024 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Shared unit test utilities.""" +import contextlib + +import os +import time +import logging +import sys + +# Run the test suite on the SDK without installing it. +sys.path.insert(0, '../') + +from time import sleep +from datetime import datetime, timedelta + +import unittest + +from utils import parse + +from splunklib import client + + + +logging.basicConfig( + filename='test.log', + level=logging.DEBUG, + format="%(asctime)s:%(levelname)s:%(message)s") + + +class NoRestartRequiredError(Exception): + pass + + +class WaitTimedOutError(Exception): + pass + + +def to_bool(x): + if x == '1': + return True + if x == '0': + return False + raise ValueError(f"Not a boolean value: {x}") + + +def tmpname(): + name = 'delete-me-' + str(os.getpid()) + str(time.time()).replace('.', '-') + return name + + +def wait(predicate, timeout=60, pause_time=0.5): + assert pause_time < timeout + start = datetime.now() + diff = timedelta(seconds=timeout) + while not predicate(): + if datetime.now() - start > diff: + logging.debug("wait timed out after %d seconds", timeout) + raise WaitTimedOutError + sleep(pause_time) + logging.debug("wait finished after %s seconds", datetime.now() - start) + + +class SDKTestCase(unittest.TestCase): + restart_already_required = False + installedApps = [] + + def assertEventuallyTrue(self, predicate, timeout=30, pause_time=0.5, + timeout_message="Operation timed out."): + assert pause_time < timeout + start = datetime.now() + diff = timedelta(seconds=timeout) + while not predicate(): + if datetime.now() - start > diff: + logging.debug("wait timed out after %d seconds", timeout) + self.fail(timeout_message) + sleep(pause_time) + logging.debug("wait finished after %s seconds", datetime.now() - start) + + def check_content(self, entity, **kwargs): + for k, v in kwargs: + self.assertEqual(entity[k], str(v)) + + def check_entity(self, entity): + assert entity is not None + self.assertTrue(entity.name is not None) + self.assertTrue(entity.path is not None) + + self.assertTrue(entity.state is not None) + self.assertTrue(entity.content is not None) + + # Verify access metadata + assert entity.access is not None + entity.access.app + entity.access.owner + entity.access.sharing + + # Verify content metadata + + # In some cases, the REST API does not return field metadata for when + # entities are intially listed by a collection, so we refresh to make + # sure the metadata is available. + entity.refresh() + + self.assertTrue(isinstance(entity.fields.required, list)) + self.assertTrue(isinstance(entity.fields.optional, list)) + self.assertTrue(isinstance(entity.fields.wildcard, list)) + + # Verify that all required fields appear in entity content + + for field in entity.fields.required: + try: + self.assertTrue(field in entity.content) + except: + # Check for known exceptions + if "configs/conf-times" in entity.path: + if field in ["is_sub_menu"]: + continue + raise + + def clear_restart_message(self): + """Tell Splunk to forget that it needs to be restarted. + + This is used mostly in cases such as deleting a temporary application. + Splunk asks to be restarted when that happens, but unless the application + contained modular input kinds or the like, it isn't necessary. + """ + if not self.service.restart_required: + raise ValueError("Tried to clear restart message when there was none.") + try: + self.service.delete("messages/restart_required") + except client.HTTPError as he: + if he.status != 404: + raise + + @contextlib.contextmanager + def fake_splunk_version(self, version): + original_version = self.service.splunk_version + try: + self.service._splunk_version = version + yield + finally: + self.service._splunk_version = original_version + + def install_app_from_collection(self, name): + collectionName = 'sdkappcollection' + if collectionName not in self.service.apps: + raise ValueError("sdk-test-application not installed in splunkd") + appPath = self.pathInApp(collectionName, ["build", name + ".tar"]) + kwargs = {"update": True, "name": appPath, "filename": True} + + try: + self.service.post("apps/local", **kwargs) + except client.HTTPError as he: + if he.status == 400: + raise IOError(f"App {name} not found in app collection") + if self.service.restart_required: + self.service.restart(120) + self.installedApps.append(name) + + def app_collection_installed(self): + collectionName = 'sdkappcollection' + return collectionName in self.service.apps + + def pathInApp(self, appName, pathComponents): + r"""Return a path to *pathComponents* in *appName*. + + `pathInApp` is used to refer to files in applications installed with + `install_app_from_collection`. For example, the app `file_to_upload` in + the collection contains `log.txt`. To get the path to it, call:: + + pathInApp('file_to_upload', ['log.txt']) + + The path to `setup.xml` in `has_setup_xml` would be fetched with:: + + pathInApp('has_setup_xml', ['default', 'setup.xml']) + + `pathInApp` figures out the correct separator to use (based on whether + splunkd is running on Windows or Unix) and joins the elements in + *pathComponents* into a path relative to the application specified by + *appName*. + + *pathComponents* should be a list of strings giving the components. + This function will try to figure out the correct separator (/ or \) + for the platform that splunkd is running on and construct the path + as needed. + + :return: A string giving the path. + """ + splunkHome = self.service.settings['SPLUNK_HOME'] + if "\\" in splunkHome: + # This clause must come first, since Windows machines may + # have mixed \ and / in their paths. + separator = "\\" + elif "/" in splunkHome: + separator = "/" + else: + raise ValueError("No separators in $SPLUNK_HOME. Can't determine what file separator to use.") + appPath = separator.join([splunkHome, "etc", "apps", appName] + pathComponents) + return appPath + + def uncheckedRestartSplunk(self, timeout=240): + self.service.restart(timeout) + + def restartSplunk(self, timeout=240): + if self.service.restart_required: + self.service.restart(timeout) + else: + raise NoRestartRequiredError() + + @classmethod + def setUpClass(cls): + cls.opts = parse([], {}, ".env") + cls.opts.kwargs.update({'retries': 3}) + # Before we start, make sure splunk doesn't need a restart. + service = client.connect(**cls.opts.kwargs) + if service.restart_required: + service.restart(timeout=120) + + def setUp(self): + unittest.TestCase.setUp(self) + self.opts.kwargs.update({'retries': 3}) + self.service = client.connect(**self.opts.kwargs) + # If Splunk is in a state requiring restart, go ahead + # and restart. That way we'll be sane for the rest of + # the test. + if self.service.restart_required: + self.restartSplunk() + logging.debug("Connected to splunkd version %s", '.'.join(str(x) for x in self.service.splunk_version)) + + def tearDown(self): + from splunklib.binding import HTTPError + + if self.service.restart_required: + self.fail("Test left Splunk in a state requiring a restart.") + + for appName in self.installedApps: + if appName in self.service.apps: + try: + self.service.apps.delete(appName) + wait(lambda: appName not in self.service.apps) + except HTTPError as error: + if not (os.name == 'nt' and error.status == 500): + raise + print(f'Ignoring failure to delete {appName} during tear down: {error}') + if self.service.restart_required: + self.clear_restart_message()