diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 974a15e2b3d0..2e2a70a9e37e 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -53,6 +53,7 @@ execution modules bluecoat_sslv bluez_bluetooth boto3_elasticache + boto3_elasticsearch boto3_route53 boto3_sns boto_apigateway diff --git a/doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst b/doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst new file mode 100644 index 000000000000..07c5408aa145 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.boto3_elasticsearch.rst @@ -0,0 +1,5 @@ +salt.modules.boto3_elasticsearch module +======================================= + +.. automodule:: salt.modules.boto3_elasticsearch + :members: diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 344cc454f4c6..45c7c1623504 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -33,6 +33,7 @@ state modules blockdev bluecoat_sslv boto3_elasticache + boto3_elasticsearch boto3_route53 boto3_sns boto_apigateway diff --git a/doc/ref/states/all/salt.states.boto3_elasticsearch.rst b/doc/ref/states/all/salt.states.boto3_elasticsearch.rst new file mode 100644 index 000000000000..5c16b2acc0a5 --- /dev/null +++ b/doc/ref/states/all/salt.states.boto3_elasticsearch.rst @@ -0,0 +1,6 @@ +salt.states.boto3_elasticsearch module +====================================== + +.. automodule:: salt.states.boto3_elasticsearch + :members: + :undoc-members: diff --git a/salt/modules/boto3_elasticsearch.py b/salt/modules/boto3_elasticsearch.py new file mode 100644 index 000000000000..b50c128e834f --- /dev/null +++ b/salt/modules/boto3_elasticsearch.py @@ -0,0 +1,1315 @@ +# -*- coding: utf-8 -*- +''' +Connection module for Amazon Elasticsearch Service + +.. versionadded:: Natrium + +:configuration: This module accepts explicit IAM credentials but can also + utilize IAM roles assigned to the instance trough Instance Profiles. + Dynamic credentials are then automatically obtained from AWS API and no + further configuration is necessary. More Information available at: + + .. code-block:: text + + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + If IAM roles are not used you need to specify them either in a pillar or + in the minion's config file: + + .. code-block:: yaml + + es.keyid: GKTADJGHEIQSXMKKRBJ08H + es.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + + A region may also be specified in the configuration: + + .. code-block:: yaml + + es.region: us-east-1 + + If a region is not specified, the default is us-east-1. + + It's also possible to specify key, keyid and region via a profile, either + as a passed in dict, or as a string to pull from pillars or minion config: + + .. code-block:: yaml + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + + All methods return a dict with: + 'result' key containing a boolean indicating success or failure, + 'error' key containing the errormessage returned by boto on error, + 'response' key containing the data of the response returned by boto on success. + +:codeauthor: Herbert Buurman +:depends: boto3 +''' +# keep lint from choking on _get_conn and _cache_id +# pylint: disable=E0602 + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import logging + +# Import Salt libs +from salt.ext import six +import salt.utils.compat +import salt.utils.json +import salt.utils.versions +from salt.exceptions import SaltInvocationError +from salt.utils.decorators import depends + +# Import third party libs + +try: + # Disable unused import-errors as these are only used for dependency checking + # pylint: disable=unused-import + import boto3 + import botocore + # pylint: enable=unused-import + from botocore.exceptions import ClientError, ParamValidationError, WaiterError + logging.getLogger('boto3').setLevel(logging.INFO) +except ImportError: + pass + +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if boto libraries exist and if boto libraries are greater than + a given version. + ''' + return salt.utils.versions.check_boto_reqs(boto3_ver='1.2.7') + + +def __init__(opts): + _ = opts + salt.utils.compat.pack_dunder(__name__) + __utils__['boto3.assign_funcs'](__name__, 'es') + + +def add_tags( + domain_name=None, + arn=None, + tags=None, + region=None, key=None, keyid=None, profile=None): + ''' + Attaches tags to an existing Elasticsearch domain. + Tags are a set of case-sensitive key value pairs. + An Elasticsearch domain may have up to 10 tags. + + :param str domain_name: The name of the Elasticsearch domain you want to add tags to. + :param str arn: The ARN of the Elasticsearch domain you want to add tags to. + Specifying this overrides ``domain_name``. + :param dict tags: The dict of tags to add to the Elasticsearch domain. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.add_tags domain_name=mydomain tags='{"foo": "bar", "baz": "qux"}' + ''' + if not any((arn, domain_name)): + raise SaltInvocationError('At least one of domain_name or arn must be specified.') + ret = {'result': False} + if arn is None: + res = describe_elasticsearch_domain( + domain_name=domain_name, + region=region, key=key, keyid=keyid, profile=profile) + if 'error' in res: + ret.update(res) + elif not res['result']: + ret.update({'error': 'The domain with name "{}" does not exist.'.format(domain_name)}) + else: + arn = res['response'].get('ARN') + if arn: + boto_params = { + 'ARN': arn, + 'TagList': [{'Key': k, 'Value': value} for k, value in six.iteritems(tags or {})] + } + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.add_tags(**boto_params) + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.12.21') +def cancel_elasticsearch_service_software_update( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Cancels a scheduled service software update for an Amazon ES domain. You can + only perform this operation before the AutomatedUpdateDate and when the UpdateStatus + is in the PENDING_UPDATE state. + + :param str domain_name: The name of the domain that you want to stop the latest + service software update on. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the current service software options. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.cancel_elasticsearch_service_software_update(DomainName=domain_name) + ret['result'] = True + res['response'] = res['ServiceSoftwareOptions'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def create_elasticsearch_domain( + domain_name, + elasticsearch_version=None, + elasticsearch_cluster_config=None, + ebs_options=None, + access_policies=None, + snapshot_options=None, + vpc_options=None, + cognito_options=None, + encryption_at_rest_options=None, + node_to_node_encryption_options=None, + advanced_options=None, + log_publishing_options=None, + blocking=False, + region=None, key=None, keyid=None, profile=None): + ''' + Given a valid config, create a domain. + + :param str domain_name: The name of the Elasticsearch domain that you are creating. + Domain names are unique across the domains owned by an account within an + AWS region. Domain names must start with a letter or number and can contain + the following characters: a-z (lowercase), 0-9, and - (hyphen). + :param str elasticsearch_version: String of format X.Y to specify version for + the Elasticsearch domain eg. "1.5" or "2.3". + :param dict elasticsearch_cluster_config: Dictionary specifying the configuration + options for an Elasticsearch domain. Keys (case sensitive) in here are: + + - InstanceType (str): The instance type for an Elasticsearch cluster. + - InstanceCount (int): The instance type for an Elasticsearch cluster. + - DedicatedMasterEnabled (bool): Indicate whether a dedicated master + node is enabled. + - ZoneAwarenessEnabled (bool): Indicate whether zone awareness is enabled. + If this is not enabled, the Elasticsearch domain will only be in one + availability zone. + - ZoneAwarenessConfig (dict): Specifies the zone awareness configuration + for a domain when zone awareness is enabled. + Keys (case sensitive) in here are: + + - AvailabilityZoneCount (int): An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. Allowed values: 2, 3 + + - DedicatedMasterType (str): The instance type for a dedicated master node. + - DedicatedMasterCount (int): Total number of dedicated master nodes, + active and on standby, for the cluster. + :param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Keys (case sensitive) in here are: + + - EBSEnabled (bool): Specifies whether EBS-based storage is enabled. + - VolumeType (str): Specifies the volume type for EBS-based storage. + - VolumeSize (int): Integer to specify the size of an EBS volume. + - Iops (int): Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :type access_policies: str or dict + :param access_policies: Dict or JSON string with the IAM access policy. + :param dict snapshot_options: Dict specifying the snapshot options. + Keys (case sensitive) in here are: + + - AutomatedSnapshotStartHour (int): Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + :param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. + Keys (case sensitive) in here are: + + - SubnetIds (list): The list of subnets for the VPC endpoint. + - SecurityGroupIds (list): The list of security groups for the VPC endpoint. + :param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. + Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Cognito for Kibana authentication. + - UserPoolId (str): Specifies the Cognito user pool ID for Kibana authentication. + - IdentityPoolId (str): Specifies the Cognito identity pool ID for Kibana authentication. + - RoleArn (str): Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + :param dict encryption_at_rest_options: Dict specifying the encryption at rest + options. Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Encryption At Rest. + - KmsKeyId (str): Specifies the KMS Key ID for Encryption At Rest options. + :param dict node_to_node_encryption_options: Dict specifying the node to node + encryption options. Keys (case sensitive) in here are: + + - Enabled (bool): Specify True to enable node-to-node encryption. + :param dict advanced_options: Dict with option to allow references to indices + in an HTTP request body. Must be False when configuring access to individual + sub-resources. By default, the value is True. + See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ + /es-createupdatedomains.html#es-createdomain-configure-advanced-options + for more information. + :param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + + - INDEX_SLOW_LOGS + - SEARCH_SLOW_LOGS + - ES_APPLICATION_LOGS + + The value assigned to each key is a dict with the following case sensitive keys: + + - CloudWatchLogsLogGroupArn (str): The ARN of the Cloudwatch log + group to which the log needs to be published. + - Enabled (bool): Specifies whether given log publishing option is enabled or not. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been created. + + Note: Not all instance types allow enabling encryption at rest. See https://docs.aws.amazon.com\ + /elasticsearch-service/latest/developerguide/aes-supported-instance-types.html + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain status configuration. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.create_elasticsearch_domain mydomain \\ + elasticsearch_cluster_config='{ \\ + "InstanceType": "t2.micro.elasticsearch", \\ + "InstanceCount": 1, \\ + "DedicatedMasterEnabled": False, \\ + "ZoneAwarenessEnabled": False}' \\ + ebs_options='{ \\ + "EBSEnabled": True, \\ + "VolumeType": "gp2", \\ + "VolumeSize": 10, \\ + "Iops": 0}' \\ + access_policies='{ \\ + "Version": "2012-10-17", \\ + "Statement": [ \\ + {"Effect": "Allow", \\ + "Principal": {"AWS": "*"}, \\ + "Action": "es:*", \\ + "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ + "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]}' \\ + snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ + advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' + ''' + boto_kwargs = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'ElasticsearchVersion': six.text_type(elasticsearch_version or ''), + 'ElasticsearchClusterConfig': elasticsearch_cluster_config, + 'EBSOptions': ebs_options, + 'AccessPolicies': (salt.utils.json.dumps(access_policies) + if isinstance(access_policies, dict) + else access_policies), + 'SnapshotOptions': snapshot_options, + 'VPCOptions': vpc_options, + 'CognitoOptions': cognito_options, + 'EncryptionAtRestOptions': encryption_at_rest_options, + 'NodeToNodeEncryptionOptions': node_to_node_encryption_options, + 'AdvancedOptions': advanced_options, + 'LogPublishingOptions': log_publishing_options, + }) + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.create_elasticsearch_domain(**boto_kwargs) + if res and 'DomainStatus' in res: + ret['result'] = True + ret['response'] = res['DomainStatus'] + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESDomainAvailable') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def delete_elasticsearch_domain( + domain_name, + blocking=False, + region=None, key=None, keyid=None, profile=None): + ''' + Permanently deletes the specified Elasticsearch domain and all of its data. + Once a domain is deleted, it cannot be recovered. + + :param str domain_name: The name of the domain to delete. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been deleted. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.delete_elasticsearch_domain(DomainName=domain_name) + ret['result'] = True + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESDomainDeleted') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.7.30') +def delete_elasticsearch_service_role( + region=None, keyid=None, key=None, profile=None): + ''' + Deletes the service-linked role that Elasticsearch Service uses to manage and + maintain VPC domains. Role deletion will fail if any existing VPC domains use + the role. You must delete any such Elasticsearch domains before deleting the role. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + conn.delete_elasticsearch_service_role() + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def describe_elasticsearch_domain( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Given a domain name gets its status description. + + :param str domain_name: The name of the domain to get the status of. + + :rtype: dict + :return: Dictionary ith key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain status information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.describe_elasticsearch_domain(DomainName=domain_name) + if res and 'DomainStatus' in res: + ret['result'] = True + ret['response'] = res['DomainStatus'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def describe_elasticsearch_domain_config( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Provides cluster configuration information about the specified Elasticsearch domain, + such as the state, creation date, update version, and update date for cluster options. + + :param str domain_name: The name of the domain to describe. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the current configuration information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.describe_elasticsearch_domain_config(DomainName=domain_name) + if res and 'DomainConfig' in res: + ret['result'] = True + ret['response'] = res['DomainConfig'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def describe_elasticsearch_domains( + domain_names, + region=None, keyid=None, key=None, profile=None): + ''' + Returns domain configuration information about the specified Elasticsearch + domains, including the domain ID, domain endpoint, and domain ARN. + + :param list domain_names: List of domain names to get information for. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the list of domain status information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.describe_elasticsearch_domains '["domain_a", "domain_b"]' + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.describe_elasticsearch_domains(DomainNames=domain_names) + if res and 'DomainStatusList' in res: + ret['result'] = True + ret['response'] = res['DomainStatusList'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.5.18') +def describe_elasticsearch_instance_type_limits( + instance_type, + elasticsearch_version, + domain_name=None, + region=None, keyid=None, key=None, profile=None): + ''' + Describe Elasticsearch Limits for a given InstanceType and ElasticsearchVersion. + When modifying existing Domain, specify the `` DomainName `` to know what Limits + are supported for modifying. + + :param str instance_type: The instance type for an Elasticsearch cluster for + which Elasticsearch ``Limits`` are needed. + :param str elasticsearch_version: Version of Elasticsearch for which ``Limits`` + are needed. + :param str domain_name: Represents the name of the Domain that we are trying + to modify. This should be present only if we are querying for Elasticsearch + ``Limits`` for existing domain. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the limits information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.describe_elasticsearch_instance_type_limits \\ + instance_type=r3.8xlarge.elasticsearch \\ + elasticsearch_version='6.2' + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'InstanceType': instance_type, + 'ElasticsearchVersion': six.text_type(elasticsearch_version), + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.describe_elasticsearch_instance_type_limits(**boto_params) + if res and 'LimitsByRole' in res: + ret['result'] = True + ret['response'] = res['LimitsByRole'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.15') +def describe_reserved_elasticsearch_instance_offerings( + reserved_elasticsearch_instance_offering_id=None, + region=None, keyid=None, key=None, profile=None): + ''' + Lists available reserved Elasticsearch instance offerings. + + :param str reserved_elasticsearch_instance_offering_id: The offering identifier + filter value. Use this parameter to show only the available offering that + matches the specified reservation identifier. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the list of offerings information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = { + 'ReservedElasticsearchInstanceOfferingId': reserved_elasticsearch_instance_offering_id + } + res = [] + for page in conn.get_paginator( + 'describe_reserved_elasticsearch_instance_offerings' + ).paginate(**boto_params): + res.extend(page['ReservedElasticsearchInstanceOfferings']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.15') +def describe_reserved_elasticsearch_instances( + reserved_elasticsearch_instance_id=None, + region=None, keyid=None, key=None, profile=None): + ''' + Returns information about reserved Elasticsearch instances for this account. + + :param str reserved_elasticsearch_instance_id: The reserved instance identifier + filter value. Use this parameter to show only the reservation that matches + the specified reserved Elasticsearch instance ID. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of information on + reserved instances. + Upon failure, also contains a key 'error' with the error message as value. + + :note: Version 1.9.174 of boto3 has a bug in that reserved_elasticsearch_instance_id + is considered a required argument, even though the documentation says otherwise. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = { + 'ReservedElasticsearchInstanceId': reserved_elasticsearch_instance_id, + } + res = [] + for page in conn.get_paginator( + 'describe_reserved_elasticsearch_instances' + ).paginate(**boto_params): + res.extend(page['ReservedElasticsearchInstances']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def get_compatible_elasticsearch_versions( + domain_name=None, + region=None, keyid=None, key=None, profile=None): + ''' + Returns a list of upgrade compatible Elastisearch versions. You can optionally + pass a ``domain_name`` to get all upgrade compatible Elasticsearch versions + for that specific domain. + + :param str domain_name: The name of an Elasticsearch domain. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of compatible versions. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.get_compatible_elasticsearch_versions(**boto_params) + if res and 'CompatibleElasticsearchVersions' in res: + ret['result'] = True + ret['response'] = res['CompatibleElasticsearchVersions'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def get_upgrade_history( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Retrieves the complete history of the last 10 upgrades that were performed on the domain. + + :param str domain_name: The name of an Elasticsearch domain. Domain names are + unique across the domains owned by an account within an AWS region. Domain + names start with a letter or number and can contain the following characters: + a-z (lowercase), 0-9, and - (hyphen). + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of upgrade histories. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = {'DomainName': domain_name} + res = [] + for page in conn.get_paginator('get_upgrade_history').paginate(**boto_params): + res.extend(page['UpgradeHistories']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def get_upgrade_status( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Retrieves the latest status of the last upgrade or upgrade eligibility check + that was performed on the domain. + + :param str domain_name: The name of an Elasticsearch domain. Domain names are + unique across the domains owned by an account within an AWS region. Domain + names start with a letter or number and can contain the following characters: + a-z (lowercase), 0-9, and - (hyphen). + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with upgrade status information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = {'DomainName': domain_name} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.get_upgrade_status(**boto_params) + ret['result'] = True + ret['response'] = res + del res['ResponseMetadata'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def list_domain_names( + region=None, keyid=None, key=None, profile=None): + ''' + Returns the name of all Elasticsearch domains owned by the current user's account. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of domain names. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.list_domain_names() + if res and 'DomainNames' in res: + ret['result'] = True + ret['response'] = [item['DomainName'] for item in res['DomainNames']] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.5.18') +def list_elasticsearch_instance_types( + elasticsearch_version, + domain_name=None, + region=None, keyid=None, key=None, profile=None): + ''' + List all Elasticsearch instance types that are supported for given ElasticsearchVersion. + + :param str elasticsearch_version: Version of Elasticsearch for which list of + supported elasticsearch instance types are needed. + :param str domain_name: DomainName represents the name of the Domain that we + are trying to modify. This should be present only if we are querying for + list of available Elasticsearch instance types when modifying existing domain. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of Elasticsearch instance types. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + boto_params = salt.utils.data.filter_falsey({ + 'ElasticsearchVersion': six.text_type(elasticsearch_version), + 'DomainName': domain_name, + }) + res = [] + for page in conn.get_paginator('list_elasticsearch_instance_types').paginate(**boto_params): + res.extend(page['ElasticsearchInstanceTypes']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.5.18') +def list_elasticsearch_versions( + region=None, keyid=None, key=None, profile=None): + ''' + List all supported Elasticsearch versions. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a list of Elasticsearch versions. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = [] + for page in conn.get_paginator('list_elasticsearch_versions').paginate(): + res.extend(page['ElasticsearchVersions']) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def list_tags( + domain_name=None, + arn=None, + region=None, key=None, keyid=None, profile=None): + ''' + Returns all tags for the given Elasticsearch domain. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with a dict of tags. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + if not any((arn, domain_name)): + raise SaltInvocationError('At least one of domain_name or arn must be specified.') + ret = {'result': False} + if arn is None: + res = describe_elasticsearch_domain( + domain_name=domain_name, + region=region, key=key, keyid=keyid, profile=profile) + if 'error' in res: + ret.update(res) + elif not res['result']: + ret.update({'error': 'The domain with name "{}" does not exist.'.format(domain_name)}) + else: + arn = res['response'].get('ARN') + if arn: + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + res = conn.list_tags(ARN=arn) + ret['result'] = True + ret['response'] = {item['Key']: item['Value'] for item in res.get('TagList', [])} + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.15') +def purchase_reserved_elasticsearch_instance_offering( + reserved_elasticsearch_instance_offering_id, + reservation_name, + instance_count=None, + region=None, keyid=None, key=None, profile=None): + ''' + Allows you to purchase reserved Elasticsearch instances. + + :param str reserved_elasticsearch_instance_offering_id: The ID of the reserved + Elasticsearch instance offering to purchase. + :param str reservation_name: A customer-specified identifier to track this reservation. + :param int instance_count: The number of Elasticsearch instances to reserve. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with purchase information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'ReservedElasticsearchInstanceOfferingId': reserved_elasticsearch_instance_offering_id, + 'ReservationName': reservation_name, + 'InstanceCount': instance_count, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.purchase_reserved_elasticsearch_instance_offering(**boto_params) + if res: + ret['result'] = True + ret['response'] = res + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def remove_tags( + tag_keys, + domain_name=None, + arn=None, + region=None, key=None, keyid=None, profile=None): + ''' + Removes the specified set of tags from the specified Elasticsearch domain. + + :param list tag_keys: List with tag keys you want to remove from the Elasticsearch domain. + :param str domain_name: The name of the Elasticsearch domain you want to remove tags from. + :param str arn: The ARN of the Elasticsearch domain you want to remove tags from. + Specifying this overrides ``domain_name``. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.remove_tags '["foo", "bar"]' domain_name=my_domain + ''' + if not any((arn, domain_name)): + raise SaltInvocationError('At least one of domain_name or arn must be specified.') + ret = {'result': False} + if arn is None: + res = describe_elasticsearch_domain( + domain_name=domain_name, + region=region, key=key, keyid=keyid, profile=profile) + if 'error' in res: + ret.update(res) + elif not res['result']: + ret.update({'error': 'The domain with name "{}" does not exist.'.format(domain_name)}) + else: + arn = res['response'].get('ARN') + if arn: + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.remove_tags(ARN=arn, + TagKeys=tag_keys) + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.12.21') +def start_elasticsearch_service_software_update( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Schedules a service software update for an Amazon ES domain. + + :param str domain_name: The name of the domain that you want to update to the + latest service software. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with service software information. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + boto_params = {'DomainName': domain_name} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.start_elasticsearch_service_software_update(**boto_params) + if res and 'ServiceSoftwareOptions' in res: + ret['result'] = True + ret['response'] = res['ServiceSoftwareOptions'] + except (ParamValidationError, ClientError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def update_elasticsearch_domain_config( + domain_name, + elasticsearch_cluster_config=None, + ebs_options=None, + vpc_options=None, + access_policies=None, + snapshot_options=None, + cognito_options=None, + advanced_options=None, + log_publishing_options=None, + blocking=False, + region=None, key=None, keyid=None, profile=None): + ''' + Modifies the cluster configuration of the specified Elasticsearch domain, + for example setting the instance type and the number of instances. + + :param str domain_name: The name of the Elasticsearch domain that you are creating. + Domain names are unique across the domains owned by an account within an + AWS region. Domain names must start with a letter or number and can contain + the following characters: a-z (lowercase), 0-9, and - (hyphen). + :param dict elasticsearch_cluster_config: Dictionary specifying the configuration + options for an Elasticsearch domain. Keys (case sensitive) in here are: + + - InstanceType (str): The instance type for an Elasticsearch cluster. + - InstanceCount (int): The instance type for an Elasticsearch cluster. + - DedicatedMasterEnabled (bool): Indicate whether a dedicated master + node is enabled. + - ZoneAwarenessEnabled (bool): Indicate whether zone awareness is enabled. + - ZoneAwarenessConfig (dict): Specifies the zone awareness configuration + for a domain when zone awareness is enabled. + Keys (case sensitive) in here are: + + - AvailabilityZoneCount (int): An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. + + - DedicatedMasterType (str): The instance type for a dedicated master node. + - DedicatedMasterCount (int): Total number of dedicated master nodes, + active and on standby, for the cluster. + :param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Keys (case sensitive) in here are: + + - EBSEnabled (bool): Specifies whether EBS-based storage is enabled. + - VolumeType (str): Specifies the volume type for EBS-based storage. + - VolumeSize (int): Integer to specify the size of an EBS volume. + - Iops (int): Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :param dict snapshot_options: Dict specifying the snapshot options. + Keys (case sensitive) in here are: + + - AutomatedSnapshotStartHour (int): Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + :param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. + Keys (case sensitive) in here are: + + - SubnetIds (list): The list of subnets for the VPC endpoint. + - SecurityGroupIds (list): The list of security groups for the VPC endpoint. + :param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. + Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Cognito for Kibana authentication. + - UserPoolId (str): Specifies the Cognito user pool ID for Kibana authentication. + - IdentityPoolId (str): Specifies the Cognito identity pool ID for Kibana authentication. + - RoleArn (str): Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + :param dict advanced_options: Dict with option to allow references to indices + in an HTTP request body. Must be False when configuring access to individual + sub-resources. By default, the value is True. + See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ + /es-createupdatedomains.html#es-createdomain-configure-advanced-options + for more information. + :param str/dict access_policies: Dict or JSON string with the IAM access policy. + :param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + + INDEX_SLOW_LOGS, SEARCH_SLOW_LOGS, ES_APPLICATION_LOGS. + + The value assigned to each key is a dict with the following case sensitive keys: + + - CloudWatchLogsLogGroupArn (str): The ARN of the Cloudwatch log + group to which the log needs to be published. + - Enabled (bool): Specifies whether given log publishing option + is enabled or not. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been updated. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain configuration. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.update_elasticsearch_domain_config mydomain \\ + elasticsearch_cluster_config='{\\ + "InstanceType": "t2.micro.elasticsearch", \\ + "InstanceCount": 1, \\ + "DedicatedMasterEnabled": false, + "ZoneAwarenessEnabled": false}' \\ + ebs_options='{\\ + "EBSEnabled": true, \\ + "VolumeType": "gp2", \\ + "VolumeSize": 10, \\ + "Iops": 0}' \\ + access_policies='{"Version": "2012-10-17", "Statement": [{\\ + "Effect": "Allow", "Principal": {"AWS": "*"}, "Action": "es:*", \\ + "Resource": "arn:aws:es:us-east-1:111111111111:domain/mydomain/*", \\ + "Condition": {"IpAddress": {"aws:SourceIp": ["127.0.0.1"]}}}]}' \\ + snapshot_options='{"AutomatedSnapshotStartHour": 0}' \\ + advanced_options='{"rest.action.multi.allow_explicit_index": "true"}' + ''' + ret = {'result': False} + boto_kwargs = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'ElasticsearchClusterConfig': elasticsearch_cluster_config, + 'EBSOptions': ebs_options, + 'SnapshotOptions': snapshot_options, + 'VPCOptions': vpc_options, + 'CognitoOptions': cognito_options, + 'AdvancedOptions': advanced_options, + 'AccessPolicies': (salt.utils.json.dumps(access_policies) + if isinstance(access_policies, dict) + else access_policies), + 'LogPublishingOptions': log_publishing_options, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.update_elasticsearch_domain_config(**boto_kwargs) + if not res or 'DomainConfig' not in res: + log.warning('Domain was not updated') + else: + ret['result'] = True + ret['response'] = res['DomainConfig'] + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESDomainAvailable') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def upgrade_elasticsearch_domain( + domain_name, + target_version, + perform_check_only=None, + blocking=False, + region=None, keyid=None, key=None, profile=None): + ''' + Allows you to either upgrade your domain or perform an Upgrade eligibility + check to a compatible Elasticsearch version. + + :param str domain_name: The name of an Elasticsearch domain. Domain names are + unique across the domains owned by an account within an AWS region. Domain + names start with a letter or number and can contain the following characters: + a-z (lowercase), 0-9, and - (hyphen). + :param str target_version: The version of Elasticsearch that you intend to + upgrade the domain to. + :param bool perform_check_only: This flag, when set to True, indicates that + an Upgrade Eligibility Check needs to be performed. This will not actually + perform the Upgrade. + :param bool blocking: Whether or not to wait (block) until the Elasticsearch + domain has been upgraded. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with the domain configuration. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.upgrade_elasticsearch_domain mydomain \\ + target_version='6.7' \\ + perform_check_only=True + ''' + ret = {'result': False} + boto_params = salt.utils.data.filter_falsey({ + 'DomainName': domain_name, + 'TargetVersion': six.text_type(target_version), + 'PerformCheckOnly': perform_check_only, + }) + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + res = conn.upgrade_elasticsearch_domain(**boto_params) + if res: + ret['result'] = True + ret['response'] = res + if blocking: + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished') + waiter.wait(DomainName=domain_name) + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def exists( + domain_name, + region=None, key=None, keyid=None, profile=None): + ''' + Given a domain name, check to see if the given domain exists. + + :param str domain_name: The name of the domain to check. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + conn.describe_elasticsearch_domain(DomainName=domain_name) + ret['result'] = True + except (ParamValidationError, ClientError) as exp: + if exp.response.get('Error', {}).get('Code') != 'ResourceNotFoundException': + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +def wait_for_upgrade( + domain_name, + region=None, keyid=None, key=None, profile=None): + ''' + Block until an upgrade-in-progress for domain ``name`` is finished. + + :param str name: The name of the domain to wait for. + + :rtype dict: + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + ''' + ret = {'result': False} + try: + conn = _get_conn(region=region, keyid=keyid, key=key, profile=profile) + waiter = __utils__['boto3_elasticsearch.get_waiter'](conn, waiter='ESUpgradeFinished') + waiter.wait(DomainName=domain_name) + ret['result'] = True + except (ParamValidationError, ClientError, WaiterError) as exp: + ret.update({'error': __utils__['boto3.get_error'](exp)['message']}) + return ret + + +@depends('botocore', version='1.10.77') +def check_upgrade_eligibility( + domain_name, + elasticsearch_version, + region=None, keyid=None, key=None, profile=None): + ''' + Helper function to determine in one call if an Elasticsearch domain can be + upgraded to the specified Elasticsearch version. + + This assumes that the Elasticsearch domain is at rest at the moment this function + is called. I.e. The domain is not in the process of : + + - being created. + - being updated. + - another upgrade running, or a check thereof. + - being deleted. + + Behind the scenes, this does 3 things: + + - Check if ``elasticsearch_version`` is among the compatible elasticsearch versions. + - Perform a check if the Elasticsearch domain is eligible for the upgrade. + - Check the result of the check and return the result as a boolean. + + :param str name: The Elasticsearch domain name to check. + :param str elasticsearch_version: The Elasticsearch version to upgrade to. + + :rtype: dict + :return: Dictionary with key 'result' and as value a boolean denoting success or failure. + Upon success, also contains a key 'reponse' with boolean result of the check. + Upon failure, also contains a key 'error' with the error message as value. + + .. versionadded:: Natrium + + CLI Example: + + .. code-block:: bash + + salt myminion boto3_elasticsearch.check_upgrade_eligibility mydomain '6.7' + ''' + ret = {'result': False} + # Check if the desired version is in the list of compatible versions + res = get_compatible_elasticsearch_versions( + domain_name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + return res + compatible_versions = res['response'][0]['TargetVersions'] + if six.text_type(elasticsearch_version) not in compatible_versions: + ret['result'] = True + ret['response'] = False + ret['error'] = ('Desired version "{}" not in compatible versions: {}.' + ''.format(elasticsearch_version, compatible_versions)) + return ret + # Check if the domain is eligible to upgrade to the desired version + res = upgrade_elasticsearch_domain( + domain_name, + elasticsearch_version, + perform_check_only=True, + blocking=True, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + return res + res = wait_for_upgrade(domain_name, region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + return res + res = get_upgrade_status(domain_name, region=region, keyid=keyid, key=key, profile=profile) + ret['result'] = True + ret['response'] = (res['response']['UpgradeStep'] == 'PRE_UPGRADE_CHECK' and + res['response']['StepStatus'] == 'SUCCEEDED') + return ret diff --git a/salt/states/boto3_elasticsearch.py b/salt/states/boto3_elasticsearch.py new file mode 100644 index 000000000000..20c1a5e088d4 --- /dev/null +++ b/salt/states/boto3_elasticsearch.py @@ -0,0 +1,754 @@ +# -*- coding: utf-8 -*- +''' +Manage Elasticsearch Service +============================ + +.. versionadded:: Natrium + +:configuration: This module accepts explicit AWS credentials but can also + utilize IAM roles assigned to the instance trough Instance Profiles. + Dynamic credentials are then automatically obtained from AWS API and no + further configuration is necessary. More Information available at: + + .. code-block:: text + + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + If IAM roles are not used you need to specify them either in a pillar or + in the minion's config file: + + .. code-block:: yaml + + es.keyid: GKTADJGHEIQSXMKKRBJ08H + es.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + + A region may also be specified in the configuration: + + .. code-block:: yaml + + es.region: us-east-1 + + If a region is not specified, the default is us-east-1. + + It's also possible to specify key, keyid and region via a profile, either + as a passed in dict, or as a string to pull from pillars or minion config: + + .. code-block:: yaml + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + +:codeauthor: Herbert Buurman +:depends: boto3 +''' + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import logging + +# Import Salt libs +import salt.utils.json +from salt.utils.versions import LooseVersion + +# Import 3rd-party libs + +log = logging.getLogger(__name__) +__virtualname__ = 'boto3_elasticsearch' + + +def __virtual__(): + ''' + Only load if boto3 and the required module functions are available. + ''' + requirements = { + 'salt': [ + 'boto3_elasticsearch.describe_elasticsearch_domain', + 'boto3_elasticsearch.create_elasticsearch_domain', + 'boto3_elasticsearch.update_elasticsearch_domain_config', + 'boto3_elasticsearch.exists', + 'boto3_elasticsearch.get_upgrade_status', + 'boto3_elasticsearch.wait_for_upgrade', + 'boto3_elasticsearch.check_upgrade_eligibility', + 'boto3_elasticsearch.upgrade_elasticsearch_domain', + ], + } + for req in requirements['salt']: + if req not in __salt__: + return (False, 'A required function was not found in __salt__: {}'.format(req)) + return __virtualname__ + + +def _check_return_value(ret): + ''' + Helper function to check if the 'result' key of the return value has been + properly set. This is to detect unexpected code-paths that would otherwise + return a 'success'-y value but not actually be succesful. + + :param dict ret: The returned value of a state function. + ''' + if ret['result'] == 'oops': + ret['result'] = False + ret['comment'].append('An internal error has occurred: The result value was ' + 'not properly changed.') + return ret + + +def present( + name, + elasticsearch_version=None, + elasticsearch_cluster_config=None, + ebs_options=None, + access_policies=None, + snapshot_options=None, + vpc_options=None, + cognito_options=None, + encryption_at_rest_options=None, + node_to_node_encryption_options=None, + advanced_options=None, + log_publishing_options=None, + blocking=True, + tags=None, + region=None, keyid=None, key=None, profile=None): + ''' + Ensure an Elasticsearch Domain exists. + + :param str name: The name of the Elasticsearch domain that you are creating. + Domain names are unique across the domains owned by an account within an + AWS region. Domain names must start with a letter or number and can contain + the following characters: a-z (lowercase), 0-9, and - (hyphen). + :param str elasticsearch_version: String of format X.Y to specify version for + the Elasticsearch domain eg. "1.5" or "2.3". + :param dict elasticsearch_cluster_config: Dict specifying the configuration + options for an Elasticsearch domain. + Keys (case sensitive) in here are: + + - InstanceType (str): The instance type for an Elasticsearch cluster. + - InstanceCount (int): The instance type for an Elasticsearch cluster. + - DedicatedMasterEnabled (bool): Indicate whether a dedicated master + node is enabled. + - ZoneAwarenessEnabled (bool): Indicate whether zone awareness is enabled. + - ZoneAwarenessConfig (dict): Specifies the zone awareness configuration + for a domain when zone awareness is enabled. + Keys (case sensitive) in here are: + + - AvailabilityZoneCount (int): An integer value to indicate the + number of availability zones for a domain when zone awareness is + enabled. This should be equal to number of subnets if VPC endpoints + is enabled. + - DedicatedMasterType (str): The instance type for a dedicated master node. + - DedicatedMasterCount (int): Total number of dedicated master nodes, + active and on standby, for the cluster. + :param dict ebs_options: Dict specifying the options to enable or disable and + specifying the type and size of EBS storage volumes. + Keys (case sensitive) in here are: + + - EBSEnabled (bool): Specifies whether EBS-based storage is enabled. + - VolumeType (str): Specifies the volume type for EBS-based storage. + - VolumeSize (int): Integer to specify the size of an EBS volume. + - Iops (int): Specifies the IOPD for a Provisioned IOPS EBS volume (SSD). + :type access_policies: str or dict + :param access_policies: Dict or JSON string with the IAM access policy. + :param dict snapshot_options: Dict specifying the snapshot options. + Keys (case senstive) in here are: + + - AutomatedSnapshotStartHour (int): Specifies the time, in UTC format, + when the service takes a daily automated snapshot of the specified + Elasticsearch domain. Default value is 0 hours. + :param dict vpc_options: Dict with the options to specify the subnets and security + groups for the VPC endpoint. + Keys (case sensitive) in here are: + + - SubnetIds (list): The list of subnets for the VPC endpoint. + - SecurityGroupIds (list): The list of security groups for the VPC endpoint. + :param dict cognito_options: Dict with options to specify the cognito user and + identity pools for Kibana authentication. + Keys (case senstive) in here are: + + - Enabled (bool): Specifies the option to enable Cognito for Kibana authentication. + - UserPoolId (str): Specifies the Cognito user pool ID for Kibana authentication. + - IdentityPoolId (str): Specifies the Cognito identity pool ID for Kibana authentication. + - RoleArn (str): Specifies the role ARN that provides Elasticsearch permissions + for accessing Cognito resources. + :param dict encryption_at_rest_options: Dict specifying the encryption at rest + options. This option can only be used for the creation of a new Elasticsearch + domain. + Keys (case sensitive) in here are: + + - Enabled (bool): Specifies the option to enable Encryption At Rest. + - KmsKeyId (str): Specifies the KMS Key ID for Encryption At Rest options. + :param dict node_to_node_encryption_options: Dict specifying the node to node + encryption options. This option can only be used for the creation of + a new Elasticsearch domain. + Keys (case sensitive) in here are: + + - Enabled (bool): Specify True to enable node-to-node encryption. + :param dict advanced_options: Dict with option to allow references to indices + in an HTTP request body. Must be False when configuring access to individual + sub-resources. By default, the value is True. + See http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide\ + /es-createupdatedomains.html#es-createdomain-configure-advanced-options + for more information. + :param dict log_publishing_options: Dict with options for various type of logs. + The keys denote the type of log file and can be one of the following: + + - INDEX_SLOW_LOGS + - SEARCH_SLOW_LOGS + - ES_APPLICATION_LOGS + + The value assigned to each key is a dict with the following case sensitive keys: + + - CloudWatchLogsLogGroupArn (str): The ARN of the Cloudwatch log + group to which the log needs to be published. + - Enabled (bool): Specifies whether given log publishing option is enabled or not. + :param bool blocking: Whether or not the state should wait for all operations + (create/update/upgrade) to be completed. Default: ``True`` + :param dict tags: Dict of tags to ensure are present on the Elasticsearch domain. + + .. versionadded:: Natrium + + Example: + + This will create an elasticsearch domain consisting of a single t2.small instance + in the eu-west-1 region (Ireland) and will wait until the instance is available + before returning from the state. + + .. code-block:: yaml + + Create new domain: + boto3_elasticsearch.present: + - name: my_domain + - elasticsearch_version: '5.1' + - elasticsearch_cluster_config: + InstanceType: t2.small.elasticsearch + InstanceCount: 1 + DedicatedMasterEnabled: False + ZoneAwarenessEnabled: False + - ebs_options: + EBSEnabled: True + VolumeType: gp2 + VolumeSize: 10 + - snapshot_options: + AutomatedSnapshotStartHour: 3 + - vpc_options: + SubnetIds: + - subnet-12345678 + SecurityGroupIds: + - sg-12345678 + - node_to_node_encryption_options: + Enabled: False + - region: eu-west-1 + - tags: + foo: bar + baz: qux + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + + action = None + current_domain = None + target_conf = salt.utils.data.filter_falsey({ + 'DomainName': name, + 'ElasticsearchClusterConfig': elasticsearch_cluster_config, + 'EBSOptions': ebs_options, + 'AccessPolicies': (salt.utils.json.dumps(access_policies) + if isinstance(access_policies, dict) + else access_policies), + 'SnapshotOptions': snapshot_options, + 'VPCOptions': vpc_options, + 'CognitoOptions': cognito_options, + 'AdvancedOptions': advanced_options, + 'LogPublishingOptions': log_publishing_options, + }, recurse_depth=3) + res = __salt__['boto3_elasticsearch.describe_elasticsearch_domain']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if not res['result']: + ret['result'] = False + if 'ResourceNotFoundException' in res['error']: + action = 'create' + config_diff = {'old': None, 'new': target_conf} + else: + ret['comment'].append(res['error']) + else: + current_domain = salt.utils.data.filter_falsey(res['response'], recurse_depth=3) + current_domain_version = current_domain['ElasticsearchVersion'] + # Remove some values from current_domain that cannot be updated + for item in ['DomainId', 'UpgradeProcessing', 'Created', 'Deleted', 'Processing', + 'Endpoints', 'ARN', 'EncryptionAtRestOptions', 'NodeToNodeEncryptionOptions', + 'ElasticsearchVersion', 'ServiceSoftwareOptions']: + if item in current_domain: + del current_domain[item] + # Further remove values from VPCOptions (if present) that are read-only + for item in ['VPCId', 'AvailabilityZones']: + if item in current_domain.get('VPCOptions', {}): + del current_domain['VPCOptions'][item] + # Some special cases + if 'CognitoOptions' in current_domain: + if 'CognitoOptions' not in target_conf and not current_domain['CognitoOptions']['Enabled']: + del current_domain['CognitoOptions'] + if 'AdvancedOptions' not in target_conf and \ + 'rest.action.multi.allow_explicit_index' in current_domain['AdvancedOptions']: + del current_domain['AdvancedOptions']['rest.action.multi.allow_explicit_index'] + if not current_domain['AdvancedOptions']: + del current_domain['AdvancedOptions'] + + # Compare current configuration with provided configuration + config_diff = salt.utils.data.recursive_diff(current_domain, target_conf) + if config_diff: + action = 'update' + + # Compare ElasticsearchVersion separately, as the update procedure differs. + if elasticsearch_version and current_domain_version != elasticsearch_version: + action = 'upgrade' + + if action in ['create', 'update']: + if __opts__['test']: + ret['result'] = None + ret['comment'].append('The Elasticsearch Domain "{}" would have been {}d.' + ''.format(name, action)) + ret['changes'] = config_diff + else: + boto_kwargs = salt.utils.data.filter_falsey({ + 'elasticsearch_version': elasticsearch_version, + 'elasticsearch_cluster_config': elasticsearch_cluster_config, + 'ebs_options': ebs_options, + 'vpc_options': vpc_options, + 'access_policies': access_policies, + 'snapshot_options': snapshot_options, + 'cognito_options': cognito_options, + 'encryption_at_rest_options': encryption_at_rest_options, + 'node_to_node_encryption_options': node_to_node_encryption_options, + 'advanced_options': advanced_options, + 'log_publishing_options': log_publishing_options, + 'blocking': blocking, + 'region': region, 'keyid': keyid, 'key': key, 'profile': profile, + }) + if action == 'update': + # Drop certain kwargs that do not apply to updates. + for item in ['elasticsearch_version', 'encryption_at_rest_options', + 'node_to_node_encryption_options']: + if item in boto_kwargs: + del boto_kwargs[item] + res = __salt__['boto3_elasticsearch.{}_elasticsearch_domain{}' + ''.format(action, '_config' if action == 'update' else '')]( + name, + **boto_kwargs) + if 'error' in res: + ret['result'] = False + ret['comment'].append(res['error']) + else: + ret['result'] = True + ret['comment'].append('Elasticsearch Domain "{}" has been {}d.'.format(name, action)) + ret['changes'] = config_diff + elif action == 'upgrade': + res = upgraded( + name, + elasticsearch_version, + region=region, keyid=keyid, key=key, profile=profile) + ret['result'] = res['result'] + ret['comment'].extend(res['comment']) + if res['changes']: + salt.utils.dictupdate.set_dict_key_value( + ret, + 'changes:old:version', + res['changes']['old']) + salt.utils.dictupdate.set_dict_key_value( + ret, + 'changes:new:version', + res['changes']['new']) + + if tags is not None: + res = tagged( + name, + tags=tags, + replace=True, + region=region, keyid=keyid, key=key, profile=profile) + ret['result'] = res['result'] + ret['comment'].extend(res['comment']) + if 'old' in res['changes']: + salt.utils.dictupdate.update_dict_key_value( + ret, + 'changes:old:tags', + res['changes']['old'] + ) + if 'new' in res['changes']: + salt.utils.dictupdate.update_dict_key_value( + ret, + 'changes:new:tags', + res['changes']['new'] + ) + ret = _check_return_value(ret) + return ret + + +def absent( + name, + blocking=True, + region=None, keyid=None, key=None, profile=None): + ''' + Ensure the Elasticsearch Domain specified does not exist. + + :param str name: The name of the Elasticsearch domain to be made absent. + :param bool blocking: Whether or not the state should wait for the deletion + to be completed. Default: ``True`` + + .. versionadded:: Natrium + + Example: + + .. code-block:: yaml + + Remove Elasticsearch Domain: + boto3_elasticsearch.absent: + - name: my_domain + - region: eu-west-1 + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + + res = __salt__['boto3_elasticsearch.exists']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append(res['error']) + elif res['result']: + if __opts__['test']: + ret['result'] = None + ret['comment'].append('Elasticsearch domain "{}" would have been removed.' + ''.format(name)) + ret['changes'] = {'old': name, 'new': None} + else: + res = __salt__['boto3_elasticsearch.delete_elasticsearch_domain']( + domain_name=name, + blocking=blocking, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error deleting Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + else: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" has been deleted.' + ''.format(name)) + ret['changes'] = {'old': name, 'new': None} + else: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" is already absent.' + ''.format(name)) + ret = _check_return_value(ret) + return ret + + +def upgraded( + name, + elasticsearch_version, + blocking=True, + region=None, keyid=None, key=None, profile=None): + ''' + Ensures the Elasticsearch domain specified runs on the specified version of + elasticsearch. Only upgrades are possible as downgrades require a manual snapshot + and an S3 bucket to store them in. + + Note that this operation is blocking until the upgrade is complete. + + :param str name: The name of the Elasticsearch domain to upgrade. + :param str elasticsearch_version: String of format X.Y to specify version for + the Elasticsearch domain eg. "1.5" or "2.3". + + .. versionadded:: Natrium + + Example: + + .. code-block:: yaml + + Upgrade Elasticsearch Domain: + boto3_elasticsearch.upgraded: + - name: my_domain + - elasticsearch_version: '7.2' + - region: eu-west-1 + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + current_domain = None + res = __salt__['boto3_elasticsearch.describe_elasticsearch_domain']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if not res['result']: + ret['result'] = False + if 'ResourceNotFoundException' in res['error']: + ret['comment'].append('The Elasticsearch domain "{}" does not exist.' + ''.format(name)) + else: + ret['comment'].append(res['error']) + else: + current_domain = res['response'] + current_version = current_domain['ElasticsearchVersion'] + if elasticsearch_version and current_version == elasticsearch_version: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" is already ' + 'at the desired version {}' + ''.format(name, elasticsearch_version)) + elif LooseVersion(elasticsearch_version) < LooseVersion(current_version): + ret['result'] = False + ret['comment'].append('Elasticsearch domain "{}" cannot be downgraded ' + 'to version "{}".' + ''.format(name, elasticsearch_version)) + if isinstance(ret['result'], bool): + return ret + log.debug('%s :upgraded: Check upgrade in progress', __name__) + # Check if an upgrade is already in progress + res = __salt__['boto3_elasticsearch.get_upgrade_status']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error determining current upgrade status ' + 'of domain "{}": {}'.format(name, res['error'])) + return ret + if res['response'].get('StepStatus') == 'IN_PROGRESS': + if blocking: + # An upgrade is already in progress, wait for it to complete + res2 = __salt__['boto3_elasticsearch.wait_for_upgrade']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res2: + ret['result'] = False + ret['comment'].append('Error waiting for upgrade of domain ' + '"{}" to complete: {}' + ''.format(name, res2['error'])) + elif res2['response'].get('UpgradeName', '').endswith(elasticsearch_version): + ret['result'] = True + ret['comment'].append('Elasticsearch Domain "{}" is ' + 'already at version "{}".' + ''.format(name, elasticsearch_version)) + else: + # We are not going to wait for it to complete, so bail. + ret['result'] = True + ret['comment'].append('An upgrade of Elasticsearch domain "{}" ' + 'is already underway: {}' + ''.format(name, res['response'].get('UpgradeName'))) + if isinstance(ret['result'], bool): + return ret + + log.debug('%s :upgraded: Check upgrade eligibility', __name__) + # Check if the domain is eligible for an upgrade + res = __salt__['boto3_elasticsearch.check_upgrade_eligibility']( + name, + elasticsearch_version, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error checking upgrade eligibility for ' + 'domain "{}": {}'.format(name, res['error'])) + elif not res['response']: + ret['result'] = False + ret['comment'].append('The Elasticsearch Domain "{}" is not eligible to ' + 'be upgraded to version {}.' + ''.format(name, elasticsearch_version)) + else: + log.debug('%s :upgraded: Start the upgrade', __name__) + # Start the upgrade + if __opts__['test']: + ret['result'] = None + ret['comment'].append('The Elasticsearch version for domain "{}" would have been upgraded.') + ret['changes'] = {'old': current_domain['ElasticsearchVersion'], + 'new': elasticsearch_version} + else: + res = __salt__['boto3_elasticsearch.upgrade_elasticsearch_domain']( + name, + elasticsearch_version, + blocking=blocking, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error upgrading Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + else: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" has been ' + 'upgraded to version {}.' + ''.format(name, elasticsearch_version)) + ret['changes'] = {'old': current_domain['ElasticsearchVersion'], + 'new': elasticsearch_version} + ret = _check_return_value(ret) + return ret + + +def latest( + name, + minor_only=True, + region=None, keyid=None, key=None, profile=None): + ''' + Ensures the Elasticsearch domain specifies runs on the latest compatible + version of elasticsearch, upgrading it if it is not. + + Note that this operation is blocking until the upgrade is complete. + + :param str name: The name of the Elasticsearch domain to upgrade. + :param bool minor_only: Only upgrade to the latest minor version. + + .. versionadded:: Natrium + + Example: + + The following example will ensure the elasticsearch domain ``my_domain`` is + upgraded to the latest minor version. So if it is currently 5.1 it will be + upgraded to 5.6. + + .. code-block:: yaml + + Upgrade Elasticsearch Domain: + boto3_elasticsearch.latest: + - name: my_domain + - minor_only: True + - region: eu-west-1 + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + # Get current version + res = __salt__['boto3_elasticsearch.describe_elasticsearch_domain']( + domain_name=name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error getting information of Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + else: + current_version = res['response']['ElasticsearchVersion'] + # Get latest compatible version + latest_version = None + res = __salt__['boto3_elasticsearch.get_compatible_elasticsearch_versions']( + domain_name=name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error getting compatible Elasticsearch versions ' + 'for Elasticsearch domain "{}": {}' + ''.format(name, res['error'])) + if isinstance(ret['result'], bool): + return ret + try: + latest_version = res['response'][0]['TargetVersions'].pop(-1) + except IndexError: + pass + if not current_version: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" can not be upgraded.' + ''.format(name)) + elif not latest_version: + ret['result'] = True + ret['comment'].append('The Elasticsearch domain "{}" is already at ' + 'the lastest version "{}".' + ''.format(name, current_version)) + else: + a_current_version = current_version.split('.') + a_latest_version = latest_version.split('.') + if not (minor_only and a_current_version[0] != a_latest_version[0]): + if __opts__['test']: + ret['result'] = None + ret['comment'].append('Elasticsearch domain "{}" would have been updated ' + 'to version "{}".'.format(name, latest_version)) + ret['changes'] = {'old': current_version, 'new': latest_version} + else: + ret = upgraded( + name, + latest_version, + region=region, keyid=keyid, key=key, profile=profile) + else: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" is already at its ' + 'latest minor version {}.' + ''.format(name, current_version)) + ret = _check_return_value(ret) + if ret['result'] and ret['changes'] and not minor_only: + # Try and see if we can upgrade again + res = latest(name, minor_only=minor_only, region=region, keyid=keyid, key=key, profile=profile) + if res['result'] and res['changes']: + ret['changes']['new'] = res['changes']['new'] + ret['comment'].extend(res['comment']) + return ret + + +def tagged( + name, + tags=None, + replace=False, + region=None, keyid=None, key=None, profile=None): + ''' + Ensures the Elasticsearch domain has the tags provided. + Adds tags to the domain unless ``replace`` is set to ``True``, in which + case all existing tags will be replaced with the tags provided in ``tags``. + (This will remove all tags if ``replace`` is ``True`` and ``tags`` is empty). + + :param str name: The Elasticsearch domain to work with. + :param dict tags: The tags to add to/replace on the Elasticsearch domain. + :param bool replace: Whether or not to replace (``True``) all existing tags + on the Elasticsearch domain, or add (``False``) tags to the ES domain. + + .. versionadded:: Natrium + + ''' + ret = {'name': name, 'result': 'oops', 'comment': [], 'changes': {}} + current_tags = {} + # Check if the domain exists + res = __salt__['boto3_elasticsearch.exists']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if res['result']: + res = __salt__['boto3_elasticsearch.list_tags']( + name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error fetching tags of Elasticsearch domain ' + '"{}": {}'.format(name, res['error'])) + else: + current_tags = res['response'] or {} + else: + ret['result'] = False + ret['comment'].append('Elasticsearch domain "{}" does not exist.' + ''.format(name)) + if isinstance(ret['result'], bool): + return ret + + diff_tags = salt.utils.dictdiffer.deep_diff(current_tags, tags) + if not diff_tags: + ret['result'] = True + ret['comment'].append('Elasticsearch domain "{}" already has the specified ' + 'tags.'.format(name)) + else: + if replace: + ret['changes'] = diff_tags + else: + ret['changes'] = {'old': current_tags, 'new': current_tags.update(tags)} + if __opts__['test']: + ret['result'] = None + ret['comment'].append('Tags on Elasticsearch domain "{}" would have ' + 'been {}ed.'.format(name, 'replac' if replace else 'add')) + else: + if replace: + res = __salt__['boto3_elasticsearch.remove_tags']( + tag_keys=current_tags.keys(), + domain_name=name, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error removing current tags from Elasticsearch ' + 'domain "{}": {}'.format(name, res['error'])) + ret['changes'] = {} + if isinstance(ret['result'], bool): + return ret + res = __salt__['boto3_elasticsearch.add_tags']( + domain_name=name, + tags=tags, + region=region, keyid=keyid, key=key, profile=profile) + if 'error' in res: + ret['result'] = False + ret['comment'].append('Error tagging Elasticsearch domain ' + '"{}": {}'.format(name, res['error'])) + ret['changes'] = {} + else: + ret['result'] = True + ret['comment'].append('Tags on Elasticsearch domain "{}" have been ' + '{}ed.'.format(name, 'replac' if replace else 'add')) + ret = _check_return_value(ret) + return ret diff --git a/salt/utils/boto3_elasticsearch.py b/salt/utils/boto3_elasticsearch.py new file mode 100644 index 000000000000..0aa84f45edb0 --- /dev/null +++ b/salt/utils/boto3_elasticsearch.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +''' +Botocore waiters for elasticsearch that are not present in boto3+botocore (yet). + +:codeauthor: Herbert Buurman +:depends: boto3 +''' +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals + +# Import Salt libs +from salt.exceptions import SaltInvocationError +import salt.utils.versions + +try: + import botocore.waiter +except ImportError: + pass + + +WAITER_CONFIGS = { + 'ESDomainAvailable': { + 'delay': 60, + 'operation': 'DescribeElasticsearchDomainConfig', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': 'Active', + 'matcher': 'path', + 'state': 'success', + 'argument': 'DomainConfig.ElasticsearchClusterConfig.Status.State', + }, { + 'expected': True, + 'matcher': 'pathAny', + 'state': 'failure', + 'argument': 'DomainConfig.*.Status.PendingDeletion', + }], + }, + 'ESUpgradeFinished': { + 'delay': 60, + 'operation': 'DescribeElasticsearchDomain', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': False, + 'matcher': 'path', + 'state': 'success', + 'argument': 'DomainStatus.UpgradeProcessing', + }], + }, + 'ESDomainDeleted': { + 'delay': 30, + 'operation': 'DescribeElasticsearchDomain', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': True, + 'matcher': 'path', + 'state': 'retry', + 'argument': 'DomainStatus.Deleted', + }, { + 'expected': False, + 'matcher': 'path', + 'state': 'failure', + 'argument': 'DomainStatus.Processing', + }, { + 'expected': 'ResourceNotFoundException', + 'matcher': 'error', + 'state': 'success', + }], + }, + 'ESDomainCreated': { + 'delay': 30, + 'operation': 'DescribeElasticsearchDomain', + 'maxAttempts': 60, + 'acceptors': [{ + 'expected': True, + 'matcher': 'path', + 'state': 'success', + 'argument': 'DomainStatus.Created', + }], + }, +} + + +def __virtual__(): + ''' + Only load if botocore libraries exist. + ''' + return salt.utils.versions.check_boto_reqs(check_boto=False) + + +def get_waiter(client, waiter=None, waiter_config=None): + ''' + Gets a botocore waiter using either one of the preconfigured models by name + ``waiter``, or with a manually supplied ``waiter_config``. + + :param botoclient client: The botocore client to use. + :param str waiter: The name of the waiter config to use. + Either ``waiter`` or ``waiter_config`` must be supplied. + If both ``waiter`` and ``waiter_config`` are supplied, ``waiter`` takes + presedence, unless no configuration for ``waiter`` exists. + :param dict waiter_config: The manual waiter config to use. + Either waiter or waiter_config must be supplied. + + :returns botocore.waiter + ''' + if not any((waiter, waiter_config)): + raise SaltInvocationError('At least one of waiter or waiter_config must be specified.') + waiter_model = botocore.waiter.WaiterModel( + {'version': 2, 'waiters': {waiter: WAITER_CONFIGS.get(waiter, waiter_config)}} + ) + return botocore.waiter.create_waiter_with_client(waiter, waiter_model, client) + + +def list_waiters(): + ''' + Lists the builtin waiter configuration names. + + :returns list + ''' + return WAITER_CONFIGS.keys() diff --git a/tests/unit/modules/test_boto3_elasticsearch.py b/tests/unit/modules/test_boto3_elasticsearch.py new file mode 100644 index 000000000000..563cee5a7e01 --- /dev/null +++ b/tests/unit/modules/test_boto3_elasticsearch.py @@ -0,0 +1,1153 @@ +# -*- coding: utf-8 -*- +''' + Tests for salt.modules.boto3_elasticsearch +''' + +# Import Python libs +from __future__ import absolute_import, print_function, unicode_literals +import random +import string +import datetime +import textwrap + +# Import Salt Testing libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import skipIf, TestCase +from tests.support.mock import (NO_MOCK, NO_MOCK_REASON, MagicMock, patch) + +# Import Salt libs +import salt.loader +from salt.utils.versions import LooseVersion +import salt.modules.boto3_elasticsearch as boto3_elasticsearch +from salt.ext.six.moves import range + +# Import 3rd-party libs +try: + import boto3 + from botocore.exceptions import ClientError + HAS_BOTO3 = True +except ImportError: + HAS_BOTO3 = False + +# the boto3_elasticsearch module relies on the connect_to_region() method +# which was added in boto 2.8.0 +# https://github.com/boto/boto/commit/33ac26b416fbb48a60602542b4ce15dcc7029f12 +REQUIRED_BOTO3_VERSION = '1.2.1' + + +def __virtual__(): + ''' + Returns True/False boolean depending on if Boto3 is installed and correct + version. + ''' + if not HAS_BOTO3: + return False + if LooseVersion(boto3.__version__) < LooseVersion(REQUIRED_BOTO3_VERSION): + return False, ('The boto3 module must be greater or equal to version {}' + ''.format(REQUIRED_BOTO3_VERSION)) + return True + + +REGION = 'us-east-1' +ACCESS_KEY = 'GKTADJGHEIQSXMKKRBJ08H' +SECRET_KEY = 'askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs' +CONN_PARAMETERS = {'region': REGION, 'key': ACCESS_KEY, 'keyid': SECRET_KEY, 'profile': {}} +ERROR_MESSAGE = 'An error occurred ({}) when calling the {} operation: Test-defined error' +ERROR_CONTENT = { + 'Error': { + 'Code': 101, + 'Message': "Test-defined error" + } +} +NOT_FOUND_ERROR = ClientError({ + 'Error': { + 'Code': 'ResourceNotFoundException', + 'Message': "Test-defined error" + } +}, 'msg') +DOMAIN_RET = { + 'DomainId': 'accountno/testdomain', + 'DomainName': 'testdomain', + 'ARN': 'arn:aws:es:region:accountno:domain/testdomain', + 'Created': True, + 'Deleted': False, + 'Endpoints': { + 'vpc': 'vpc-testdomain-1234567890.region.es.amazonaws.com' + }, + 'Processing': False, + 'UpgradeProcessing': False, + 'ElasticsearchVersion': '6.3', + 'ElasticsearchClusterConfig': { + 'InstanceType': 't2.medium.elasticsearch', + 'InstanceCount': 1, + 'DedicatedMasterEnabled': False, + 'ZoneAwarenessEnabled': False, + }, + 'EBSOptions': { + 'EBSEnabled': True, + 'VolumeType': 'gp2', + 'VolumeSize': 123, + 'Iops': 12 + }, + 'AccessPolicies': textwrap.dedent(''' + {"Version":"2012-10-17","Statement":[{"Effect":"Allow", + "Principal":{"AWS":"*"},"Action":"es:*", + "Resource":"arn:aws:es:region:accountno:domain/testdomain/*"}]}'''), + 'SnapshotOptions': { + 'AutomatedSnapshotStartHour': 1 + }, + 'VPCOptions': { + 'VPCId': 'vpc-12345678', + 'SubnetIds': [ + 'subnet-deadbeef', + ], + 'AvailabilityZones': [ + 'regiona', + ], + 'SecurityGroupIds': [ + 'sg-87654321', + ] + }, + 'CognitoOptions': { + 'Enabled': False, + }, + 'EncryptionAtRestOptions': { + 'Enabled': False, + }, + 'NodeToNodeEncryptionOptions': { + 'Enabled': False + }, + 'AdvancedOptions': { + 'rest.action.multi.allow_explicit_index': 'true' + }, + 'ServiceSoftwareOptions': { + 'CurrentVersion': 'R20190221-P1', + 'NewVersion': 'R20190418', + 'UpdateAvailable': True, + 'Cancellable': False, + 'UpdateStatus': 'ELIGIBLE', + 'Description': ('A newer release R20190418 is available. This release ' + 'will be automatically deployed after somedate'), + 'AutomatedUpdateDate': None + } +} + + +@skipIf(HAS_BOTO3 is False, 'The boto module must be installed.') +@skipIf(LooseVersion(boto3.__version__) < LooseVersion(REQUIRED_BOTO3_VERSION), + 'The boto3 module must be greater or equal to version {}'.format(REQUIRED_BOTO3_VERSION)) +@skipIf(NO_MOCK, NO_MOCK_REASON) +class Boto3ElasticsearchTestCase(TestCase, LoaderModuleMockMixin): + ''' + TestCase for salt.modules.boto3_elasticsearch module + ''' + conn = None + + def setup_loader_modules(self): + self.opts = salt.config.DEFAULT_MINION_OPTS.copy() + utils = salt.loader.utils( + self.opts, + whitelist=['boto3', 'args', 'systemd', 'path', 'platform'], + context={}) + return {boto3_elasticsearch: {'__utils__': utils}} + + def setUp(self): + super(Boto3ElasticsearchTestCase, self).setUp() + boto3_elasticsearch.__init__(self.opts) + del self.opts + + # Set up MagicMock to replace the boto3 session + # connections keep getting cached from prior tests, can't find the + # correct context object to clear it. So randomize the cache key, to prevent any + # cache hits + CONN_PARAMETERS['key'] = ''.join(random.choice(string.ascii_lowercase + string.digits) + for _ in range(50)) + + self.conn = MagicMock() + self.addCleanup(delattr, self, 'conn') + self.patcher = patch('boto3.session.Session') + self.addCleanup(self.patcher.stop) + self.addCleanup(delattr, self, 'patcher') + mock_session = self.patcher.start() + session_instance = mock_session.return_value + session_instance.configure_mock(client=MagicMock(return_value=self.conn)) + self.paginator = MagicMock() + self.addCleanup(delattr, self, 'paginator') + self.conn.configure_mock(get_paginator=MagicMock(return_value=self.paginator)) + + def test_describe_elasticsearch_domain_positive(self): + ''' + Test that when describing a domain when the domain actually exists, + the .exists method returns a dict with 'result': True + and 'response' with the domain status information. + ''' + # The patch below is not neccesary per se, + # as .exists returns positive as long as no exception is raised. + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_domain( + domain_name='testdomain', + **CONN_PARAMETERS), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_describe_elasticsearch_domain_error(self): + ''' + Test that when describing a domain when the domain does not exist, + the .exists method returns a dict with 'result': False + and 'error' with boto's ResourceNotFoundException. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + side_effect=NOT_FOUND_ERROR): + result = boto3_elasticsearch.describe_elasticsearch_domain( + domain_name='testdomain', + **CONN_PARAMETERS) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format('ResourceNotFoundException', 'msg') + ) + self.assertFalse(result['result']) + + def test_create_elasticsearch_domain_positive(self): + ''' + Test that when creating a domain, and it succeeds, + the .create method returns a dict with 'result': True + and 'response' with the newly created domain's status information. + ''' + with patch.object(self.conn, + 'create_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + kwargs = { + 'elasticsearch_version': DOMAIN_RET['ElasticsearchVersion'], + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'encryption_at_rest_options': DOMAIN_RET['EncryptionAtRestOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + } + kwargs.update(CONN_PARAMETERS) + self.assertEqual( + boto3_elasticsearch.create_elasticsearch_domain(domain_name='testdomain', **kwargs), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_create_elasticsearch_domain_error(self): + ''' + Test that when creating a domain, and boto3 returns an error, + the .create method returns a dict with 'result': False + and 'error' with the error reported by boto3. + ''' + with patch.object(self.conn, + 'create_elasticsearch_domain', + side_effect=ClientError(ERROR_CONTENT, 'create_domain')): + kwargs = { + 'elasticsearch_version': DOMAIN_RET['ElasticsearchVersion'], + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'encryption_at_rest_options': DOMAIN_RET['EncryptionAtRestOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + } + kwargs.update(CONN_PARAMETERS) + result = boto3_elasticsearch.create_elasticsearch_domain('testdomain', **kwargs) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'create_domain') + ) + + def test_delete_domain_positive(self): + ''' + Test that when deleting a domain, and it succeeds, + the .delete method returns {'result': True}. + ''' + with patch.object(self.conn, 'delete_elasticsearch_domain'): + self.assertEqual( + boto3_elasticsearch.delete_elasticsearch_domain('testdomain', **CONN_PARAMETERS), + {'result': True} + ) + + def test_delete_domain_error(self): + ''' + Test that when deleting a domain, and boto3 returns an error, + the .delete method returns {'result': False, 'error' :'the error'}. + ''' + with patch.object(self.conn, + 'delete_elasticsearch_domain', + side_effect=ClientError(ERROR_CONTENT, 'delete_domain')): + result = boto3_elasticsearch.delete_elasticsearch_domain('testdomain', **CONN_PARAMETERS) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'delete_domain') + ) + + def test_update_domain_positive(self): + ''' + Test that when updating a domain succeeds, the .update method returns {'result': True}. + ''' + with patch.object(self.conn, + 'update_elasticsearch_domain_config', + return_value={'DomainConfig': DOMAIN_RET}): + kwargs = { + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'log_publishing_options': {}, + } + + kwargs.update(CONN_PARAMETERS) + self.assertEqual( + boto3_elasticsearch.update_elasticsearch_domain_config('testdomain', **kwargs), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_update_domain_error(self): + ''' + Test that when updating a domain fails, and boto3 returns an error, + the .update method returns the error. + ''' + with patch.object(self.conn, + 'update_elasticsearch_domain_config', + side_effect=ClientError(ERROR_CONTENT, 'update_domain')): + kwargs = { + 'elasticsearch_cluster_config': DOMAIN_RET['ElasticsearchClusterConfig'], + 'ebs_options': DOMAIN_RET['EBSOptions'], + 'snapshot_options': DOMAIN_RET['SnapshotOptions'], + 'vpc_options': DOMAIN_RET['VPCOptions'], + 'cognito_options': DOMAIN_RET['CognitoOptions'], + 'advanced_options': DOMAIN_RET['AdvancedOptions'], + 'access_policies': DOMAIN_RET['AccessPolicies'], + 'log_publishing_options': {}, + } + kwargs.update(CONN_PARAMETERS) + result = boto3_elasticsearch.update_elasticsearch_domain_config('testdomain', **kwargs) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'update_domain') + ) + + def test_add_tags_positive(self): + ''' + Test that when adding tags is succesful, the .add_tags method returns {'result': True}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.add_tags( + 'testdomain', + tags={'foo': 'bar', 'baz': 'qux'}, + **CONN_PARAMETERS + ), + {'result': True} + ) + + def test_add_tags_error(self): + ''' + Test that when adding tags fails, and boto3 returns an error, + the .add_tags function returns {'tagged': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'add_tags', + side_effect=ClientError(ERROR_CONTENT, 'add_tags')), \ + patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + result = boto3_elasticsearch.add_tags( + 'testdomain', + tags={'foo': 'bar', 'baz': 'qux'}, + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'add_tags') + ) + + def test_remove_tags_positive(self): + ''' + Test that when removing tags is succesful, the .remove_tags method returns {'tagged': True}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.remove_tags( + tag_keys=['foo', 'bar'], + domain_name='testdomain', + **CONN_PARAMETERS), + {'result': True} + ) + + def test_remove_tag_error(self): + ''' + Test that when removing tags fails, and boto3 returns an error, + the .remove_tags method returns {'tagged': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'remove_tags', + side_effect=ClientError(ERROR_CONTENT, 'remove_tags')), \ + patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + result = boto3_elasticsearch.remove_tags( + tag_keys=['foo', 'bar'], + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'remove_tags') + ) + + def test_list_tags_positive(self): + ''' + Test that when listing tags is succesful, + the .list_tags method returns a dict with key 'tags'. + Also test that the tags returned are manipulated properly (i.e. transformed + into a dict with tags). + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}), \ + patch.object(self.conn, + 'list_tags', + return_value={'TagList': [{'Key': 'foo', 'Value': 'bar'}]}): + result = boto3_elasticsearch.list_tags( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertEqual(result, { + 'result': True, + 'response': {'foo': 'bar'} + }) + + def test_list_tags_error(self): + ''' + Test that when listing tags causes boto3 to return an error, + the .list_tags method returns the error. + ''' + with patch.object(self.conn, + 'list_tags', + side_effect=ClientError(ERROR_CONTENT, 'list_tags')), \ + patch.object(self.conn, + 'describe_elasticsearch_domain', + return_value={'DomainStatus': DOMAIN_RET}): + result = boto3_elasticsearch.list_tags( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'list_tags') + ) + + def test_cancel_elasticsearch_service_software_update_positive(self): + ''' + Test that when calling cancel_elasticsearch_service_software_update and + it is succesful, it returns {'result': True}. + ''' + retval = { + 'ServiceSoftwareOptions': { + 'CurrentVersion': 'string', + 'NewVersion': 'string', + 'UpdateAvailable': True, + 'Cancellable': True, + 'UpdateStatus': 'ELIGIBLE', + 'Description': 'string', + 'AutomatedUpdateDate': datetime.datetime(2015, 1, 1), + } + } + with patch.object(self.conn, + 'cancel_elasticsearch_service_software_update', + return_value=retval): + result = boto3_elasticsearch.cancel_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertEqual(result, { + 'result': True, + }) + + def test_cancel_elasticsearch_service_software_update_error(self): + ''' + Test that when calling cancel_elasticsearch_service_software_update and + boto3 returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'cancel_elasticsearch_service_software_update', + side_effect=ClientError(ERROR_CONTENT, 'cancel_elasticsearch_service_software_update')): + result = boto3_elasticsearch.cancel_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'cancel_elasticsearch_service_software_update') + ) + + def test_delete_elasticsearch_service_role_positive(self): + ''' + Test that when calling delete_elasticsearch_service_role and + it is succesful, it returns {'result': True}. + ''' + with patch.object(self.conn, + 'delete_elasticsearch_service_role', + return_value=None): + result = boto3_elasticsearch.delete_elasticsearch_service_role( + **CONN_PARAMETERS + ) + self.assertEqual(result, { + 'result': True, + }) + + def test_delete_elasticsearch_service_role_error(self): + ''' + Test that when calling delete_elasticsearch_service_role and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'delete_elasticsearch_service_role', + side_effect=ClientError(ERROR_CONTENT, 'delete_elasticsearch_service_role')): + result = boto3_elasticsearch.delete_elasticsearch_service_role( + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'delete_elasticsearch_service_role') + ) + + def test_describe_elasticsearch_domain_config_positive(self): + ''' + Test that when calling describe_elasticsearch_domain_config and + it is succesful, it returns {'result': True}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain_config', + return_value={'DomainConfig': DOMAIN_RET}): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_domain_config('testdomain', **CONN_PARAMETERS), + {'result': True, 'response': DOMAIN_RET} + ) + + def test_describe_elasticsearch_domain_config_error(self): + ''' + Test that when calling describe_elasticsearch_domain_config and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domain_config', + side_effect=ClientError(ERROR_CONTENT, 'describe_elasticsearch_domain_config')): + result = boto3_elasticsearch.describe_elasticsearch_domain_config( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'describe_elasticsearch_domain_config') + ) + + def test_describe_elasticsearch_domains_positive(self): + ''' + Test that when calling describe_elasticsearch_domains and it is succesful, + it returns {'result': True, 'response': some_data}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domains', + return_value={'DomainStatusList': [DOMAIN_RET]}): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_domains( + domain_names=['test_domain'], + **CONN_PARAMETERS + ), + {'result': True, 'response': [DOMAIN_RET]} + ) + + def test_describe_elasticsearch_domains_error(self): + ''' + Test that when calling describe_elasticsearch_domains and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_domains', + side_effect=ClientError(ERROR_CONTENT, 'describe_elasticsearch_domains')): + result = boto3_elasticsearch.describe_elasticsearch_domains( + domain_names=['testdomain'], + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'describe_elasticsearch_domains') + ) + + def test_describe_elasticsearch_instance_type_limits_positive(self): + ''' + Test that when calling describe_elasticsearch_instance_type_limits and + it succeeds, it returns {'result': True, 'response' some_value}. + ''' + ret_val = { + 'LimitsByRole': { + 'string': { + 'StorageTypes': [{ + 'StorageTypeName': 'string', + 'StorageSubTypeName': 'string', + 'StorageTypeLimits': [{ + 'LimitName': 'string', + 'LimitValues': ['string'], + }], + }], + 'InstanceLimits': { + 'InstanceCountLimits': { + 'MinimumInstanceCount': 123, + 'MaximumInstanceCount': 123 + } + }, + 'AdditionalLimits': [{ + 'LimitName': 'string', + 'LimitValues': ['string'] + }], + } + } + } + with patch.object(self.conn, + 'describe_elasticsearch_instance_type_limits', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.describe_elasticsearch_instance_type_limits( + domain_name='testdomain', + instance_type='foo', + elasticsearch_version='1.0', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['LimitsByRole']} + ) + + def test_describe_elasticsearch_instance_type_limits_error(self): + ''' + Test that when calling describe_elasticsearch_instance_type_limits and boto3 returns + an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'describe_elasticsearch_instance_type_limits', + side_effect=ClientError(ERROR_CONTENT, 'describe_elasticsearch_instance_type_limits')): + result = boto3_elasticsearch.describe_elasticsearch_instance_type_limits( + domain_name='testdomain', + instance_type='foo', + elasticsearch_version='1.0', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'describe_elasticsearch_instance_type_limits') + ) + + def test_describe_reserved_elasticsearch_instance_offerings_positive(self): + ''' + Test that when calling describe_reserved_elasticsearch_instance_offerings + and it succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'NextToken': 'string', + 'ReservedElasticsearchInstanceOfferings': [{ + 'ReservedElasticsearchInstanceOfferingId': 'string', + 'ElasticsearchInstanceType': 't2.medium.elasticsearch', + 'Duration': 123, + 'FixedPrice': 123.0, + 'UsagePrice': 123.0, + 'CurrencyCode': 'string', + 'PaymentOption': 'NO_UPFRONT', + 'RecurringCharges': [{ + 'RecurringChargeAmount': 123.0, + 'RecurringChargeFrequency': 'string' + }] + }] + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.describe_reserved_elasticsearch_instance_offerings( + reserved_elasticsearch_instance_offering_id='foo', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ReservedElasticsearchInstanceOfferings']} + ) + + def test_describe_reserved_elasticsearch_instance_offerings_error(self): + ''' + Test that when calling describe_reserved_elasticsearch_instance_offerings + and boto3 returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'describe_reserved_elasticsearch_instance_offerings')): + result = boto3_elasticsearch.describe_reserved_elasticsearch_instance_offerings( + reserved_elasticsearch_instance_offering_id='foo', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'describe_reserved_elasticsearch_instance_offerings') + ) + + def test_describe_reserved_elasticsearch_instances_positive(self): + ''' + Test that when calling describe_reserved_elasticsearch_instances and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'NextToken': 'string', + 'ReservedElasticsearchInstances': [{ + 'ReservationName': 'string', + 'ReservedElasticsearchInstanceId': 'string', + 'ReservedElasticsearchInstanceOfferingId': 'string', + 'ElasticsearchInstanceType': 't2.medium.elasticsearch', + 'StartTime': datetime.datetime(2015, 1, 1), + 'Duration': 123, + 'FixedPrice': 123.0, + 'UsagePrice': 123.0, + 'CurrencyCode': 'string', + 'ElasticsearchInstanceCount': 123, + 'State': 'string', + 'PaymentOption': 'ALL_UPFRONT', + 'RecurringCharges': [{ + 'RecurringChargeAmount': 123.0, + 'RecurringChargeFrequency': 'string' + }, + ] + }, + ] + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.describe_reserved_elasticsearch_instances( + reserved_elasticsearch_instance_id='foo', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ReservedElasticsearchInstances']} + ) + + def test_describe_reserved_elasticsearch_instances_error(self): + ''' + Test that when calling describe_reserved_elasticsearch_instances and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'describe_reserved_elasticsearch_instances')): + result = boto3_elasticsearch.describe_reserved_elasticsearch_instances( + reserved_elasticsearch_instance_id='foo', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'describe_reserved_elasticsearch_instances') + ) + + def test_get_compatible_elasticsearch_versions_positive(self): + ''' + Test that when calling get_compatible_elasticsearch_versions and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'CompatibleElasticsearchVersions': [{ + 'SourceVersion': 'string', + 'TargetVersions': [ + 'string', + ] + }] + } + with patch.object(self.conn, + 'get_compatible_elasticsearch_versions', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.get_compatible_elasticsearch_versions( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['CompatibleElasticsearchVersions']} + ) + + def test_get_compatible_elasticsearch_versions_error(self): + ''' + Test that when calling get_compatible_elasticsearch_versions and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'get_compatible_elasticsearch_versions', + side_effect=ClientError(ERROR_CONTENT, 'get_compatible_elasticsearch_versions')): + result = boto3_elasticsearch.get_compatible_elasticsearch_versions( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'get_compatible_elasticsearch_versions') + ) + + def test_get_upgrade_history_positive(self): + ''' + Test that when calling get_upgrade_history and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'UpgradeHistories': [{ + 'UpgradeName': 'string', + 'StartTimestamp': datetime.datetime(2015, 1, 1), + 'UpgradeStatus': 'IN_PROGRESS', + 'StepsList': [{ + 'UpgradeStep': 'PRE_UPGRADE_CHECK', + 'UpgradeStepStatus': 'IN_PROGRESS', + 'Issues': [ + 'string', + ], + 'ProgressPercent': 123.0 + }] + }], + 'NextToken': 'string' + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.get_upgrade_history( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['UpgradeHistories']} + ) + + def test_get_upgrade_history_error(self): + ''' + Test that when calling get_upgrade_history and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'get_upgrade_history')): + result = boto3_elasticsearch.get_upgrade_history( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'get_upgrade_history') + ) + + def test_get_upgrade_status_positive(self): + ''' + Test that when calling get_upgrade_status and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'UpgradeStep': 'PRE_UPGRADE_CHECK', + 'StepStatus': 'IN_PROGRESS', + 'UpgradeName': 'string', + 'ResponseMetadata': None, + } + with patch.object(self.conn, + 'get_upgrade_status', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.get_upgrade_status( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val} + ) + + def test_get_upgrade_status_error(self): + ''' + Test that when calling get_upgrade_status and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'get_upgrade_status', + side_effect=ClientError(ERROR_CONTENT, 'get_upgrade_status')): + result = boto3_elasticsearch.get_upgrade_status( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'get_upgrade_status') + ) + + def test_list_domain_names_positive(self): + ''' + Test that when calling list_domain_names and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'DomainNames': [{ + 'DomainName': 'string' + }] + } + with patch.object(self.conn, + 'list_domain_names', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.list_domain_names( + **CONN_PARAMETERS + ), + {'result': True, 'response': [item['DomainName'] for item in ret_val['DomainNames']]} + ) + + def test_list_domain_names_error(self): + ''' + Test that when calling list_domain_names and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'list_domain_names', + side_effect=ClientError(ERROR_CONTENT, 'list_domain_names')): + result = boto3_elasticsearch.list_domain_names( + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'list_domain_names') + ) + + def test_list_elasticsearch_instance_types_positive(self): + ''' + Test that when calling list_elasticsearch_instance_types and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ElasticsearchInstanceTypes': [ + 'm3.medium.elasticsearch', 'm3.large.elasticsearch', 'm3.xlarge.elasticsearch', + 'm3.2xlarge.elasticsearch', 'm4.large.elasticsearch', 'm4.xlarge.elasticsearch', + 'm4.2xlarge.elasticsearch', 'm4.4xlarge.elasticsearch', 'm4.10xlarge.elasticsearch', + 't2.micro.elasticsearch', 't2.small.elasticsearch', 't2.medium.elasticsearch', + 'r3.large.elasticsearch', 'r3.xlarge.elasticsearch', 'r3.2xlarge.elasticsearch', + 'r3.4xlarge.elasticsearch', 'r3.8xlarge.elasticsearch', 'i2.xlarge.elasticsearch', + 'i2.2xlarge.elasticsearch', 'd2.xlarge.elasticsearch', 'd2.2xlarge.elasticsearch', + 'd2.4xlarge.elasticsearch', 'd2.8xlarge.elasticsearch', 'c4.large.elasticsearch', + 'c4.xlarge.elasticsearch', 'c4.2xlarge.elasticsearch', 'c4.4xlarge.elasticsearch', + 'c4.8xlarge.elasticsearch', 'r4.large.elasticsearch', 'r4.xlarge.elasticsearch', + 'r4.2xlarge.elasticsearch', 'r4.4xlarge.elasticsearch', 'r4.8xlarge.elasticsearch', + 'r4.16xlarge.elasticsearch', 'i3.large.elasticsearch', 'i3.xlarge.elasticsearch', + 'i3.2xlarge.elasticsearch', 'i3.4xlarge.elasticsearch', 'i3.8xlarge.elasticsearch', + 'i3.16xlarge.elasticsearch', + ], + 'NextToken': 'string' + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.list_elasticsearch_instance_types( + elasticsearch_version='1.0', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ElasticsearchInstanceTypes']} + ) + + def test_list_elasticsearch_instance_types_error(self): + ''' + Test that when calling list_elasticsearch_instance_types and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'list_elasticsearch_instance_types')): + result = boto3_elasticsearch.list_elasticsearch_instance_types( + elasticsearch_version='1.0', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'list_elasticsearch_instance_types') + ) + + def test_list_elasticsearch_versions_positive(self): + ''' + Test that when calling list_elasticsearch_versions and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ElasticsearchVersions': ['string'], + 'NextToken': 'string' + } + with patch.object(self.paginator, + 'paginate', + return_value=[ret_val]): + self.assertEqual( + boto3_elasticsearch.list_elasticsearch_versions( + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ElasticsearchVersions']} + ) + + def test_list_elasticsearch_versions_error(self): + ''' + Test that when calling list_elasticsearch_versions and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.paginator, + 'paginate', + side_effect=ClientError(ERROR_CONTENT, 'list_elasticsearch_versions')): + result = boto3_elasticsearch.list_elasticsearch_versions( + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'list_elasticsearch_versions') + ) + + def test_purchase_reserved_elasticsearch_instance_offering_positive(self): + ''' + Test that when calling purchase_reserved_elasticsearch_instance_offering and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ReservedElasticsearchInstanceId': 'string', + 'ReservationName': 'string' + } + with patch.object(self.conn, + 'purchase_reserved_elasticsearch_instance_offering', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.purchase_reserved_elasticsearch_instance_offering( + reserved_elasticsearch_instance_offering_id='foo', + reservation_name='bar', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val} + ) + + def test_purchase_reserved_elasticsearch_instance_offering_error(self): + ''' + Test that when calling purchase_reserved_elasticsearch_instance_offering and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'purchase_reserved_elasticsearch_instance_offering', + side_effect=ClientError(ERROR_CONTENT, 'purchase_reserved_elasticsearch_instance_offering')): + result = boto3_elasticsearch.purchase_reserved_elasticsearch_instance_offering( + reserved_elasticsearch_instance_offering_id='foo', + reservation_name='bar', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'purchase_reserved_elasticsearch_instance_offering') + ) + + def test_start_elasticsearch_service_software_update_positive(self): + ''' + Test that when calling start_elasticsearch_service_software_update and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'ServiceSoftwareOptions': { + 'CurrentVersion': 'string', + 'NewVersion': 'string', + 'UpdateAvailable': True, + 'Cancellable': True, + 'UpdateStatus': 'PENDING_UPDATE', + 'Description': 'string', + 'AutomatedUpdateDate': datetime.datetime(2015, 1, 1) + } + } + with patch.object(self.conn, + 'start_elasticsearch_service_software_update', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.start_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val['ServiceSoftwareOptions']} + ) + + def test_start_elasticsearch_service_software_update_error(self): + ''' + Test that when calling start_elasticsearch_service_software_update and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'start_elasticsearch_service_software_update', + side_effect=ClientError(ERROR_CONTENT, 'start_elasticsearch_service_software_update')): + result = boto3_elasticsearch.start_elasticsearch_service_software_update( + domain_name='testdomain', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'start_elasticsearch_service_software_update') + ) + + def test_upgrade_elasticsearch_domain_positive(self): + ''' + Test that when calling upgrade_elasticsearch_domain and it + succeeds, it returns {'result': True, 'response': some_value}. + ''' + ret_val = { + 'DomainName': 'string', + 'TargetVersion': 'string', + 'PerformCheckOnly': True + } + with patch.object(self.conn, + 'upgrade_elasticsearch_domain', + return_value=ret_val): + self.assertEqual( + boto3_elasticsearch.upgrade_elasticsearch_domain( + domain_name='testdomain', + target_version='1.1', + **CONN_PARAMETERS + ), + {'result': True, 'response': ret_val} + ) + + def test_upgrade_elasticsearch_domain_error(self): + ''' + Test that when calling upgrade_elasticsearch_domain and boto3 + returns an error, it returns {'result': False, 'error': 'the error'}. + ''' + with patch.object(self.conn, + 'upgrade_elasticsearch_domain', + side_effect=ClientError(ERROR_CONTENT, 'upgrade_elasticsearch_domain')): + result = boto3_elasticsearch.upgrade_elasticsearch_domain( + domain_name='testdomain', + target_version='1.1', + **CONN_PARAMETERS + ) + self.assertFalse(result['result']) + self.assertEqual( + result.get('error', ''), + ERROR_MESSAGE.format(101, 'upgrade_elasticsearch_domain') + )