Merge branch 'release-1.23.41'

* release-1.23.41:
  Bumping version to 1.23.41
  Update to latest models
  Add tests and function cleanup
  Mask proxy url credentials

.changes/1.23.41.json

@@ -0,0 +1,17 @@
+[
+  {
+    "category": "Exceptions",
+    "description": "ProxyConnectionError previously provided the full proxy URL. User info will now be appropriately masked if needed.",
+    "type": "enhancement"
+  },
+  {
+    "category": "``mediaconvert``",
+    "description": "AWS Elemental MediaConvert SDK has added support for 4K AV1 output resolutions & 10-bit AV1 color, the ability to ingest sidecar Dolby Vision XML metadata files, and the ability to flag WebVTT and IMSC tracks for accessibility in HLS.",
+    "type": "api-change"
+  },
+  {
+    "category": "``transcribe``",
+    "description": "Add support for granular PIIEntityTypes when using Batch ContentRedaction.",
+    "type": "api-change"
+  }
+]

CHANGELOG.rst

@@ -2,6 +2,14 @@
 CHANGELOG
 =========
 
+1.23.41
+=======
+
+* enhancement:Exceptions: ProxyConnectionError previously provided the full proxy URL. User info will now be appropriately masked if needed.
+* api-change:``mediaconvert``: AWS Elemental MediaConvert SDK has added support for 4K AV1 output resolutions & 10-bit AV1 color, the ability to ingest sidecar Dolby Vision XML metadata files, and the ability to flag WebVTT and IMSC tracks for accessibility in HLS.
+* api-change:``transcribe``: Add support for granular PIIEntityTypes when using Batch ContentRedaction.
+
+
 1.23.40
 =======
 

botocore/__init__.py

@@ -16,7 +16,7 @@
 import re
 import logging
 
-__version__ = '1.23.40'
+__version__ = '1.23.41'
 
 
 class NullHandler(logging.Handler):

botocore/data/transcribe/2017-10-26/service-2.json

@@ -897,6 +897,10 @@
         "RedactionOutput":{
           "shape":"RedactionOutput",
           "documentation":"<p>The output transcript file stored in either the default S3 bucket or in a bucket you specify.</p> <p>When you choose <code>redacted</code> Amazon Transcribe outputs only the redacted transcript.</p> <p>When you choose <code>redacted_and_unredacted</code> Amazon Transcribe outputs both the redacted and unredacted transcripts.</p>"
+        },
+        "PiiEntityTypes":{
+          "shape":"PiiEntityTypes",
+          "documentation":"<p>The types of personally identifiable information (PII) you want to redact in your transcript.</p>"
         }
       },
       "documentation":"<p>Settings for content redaction within a transcription job.</p>"
@@ -2269,6 +2273,29 @@
       "type":"list",
       "member":{"shape":"Phrase"}
     },
+    "PiiEntityType":{
+      "type":"string",
+      "enum":[
+        "BANK_ACCOUNT_NUMBER",
+        "BANK_ROUTING",
+        "CREDIT_DEBIT_NUMBER",
+        "CREDIT_DEBIT_CVV",
+        "CREDIT_DEBIT_EXPIRY",
+        "PIN",
+        "EMAIL",
+        "ADDRESS",
+        "NAME",
+        "PHONE",
+        "SSN",
+        "ALL"
+      ]
+    },
+    "PiiEntityTypes":{
+      "type":"list",
+      "member":{"shape":"PiiEntityType"},
+      "max":11,
+      "min":0
+    },
     "RedactionOutput":{
       "type":"string",
       "enum":[

botocore/httpsession.py

@@ -148,6 +148,24 @@ def ensure_boolean(val):
         return val.lower() == 'true'
 
 
+def mask_proxy_url(proxy_url):
+    """
+    Mask proxy url credentials.
+
+    :type proxy_url: str
+    :param proxy_url: The proxy url, i.e. https://username:password@proxy.com
+
+    :return: Masked proxy url, i.e. https://***:***@proxy.com
+    """
+    mask = '*' * 3
+    parsed_url = urlparse(proxy_url)
+    if parsed_url.username:
+        proxy_url = proxy_url.replace(parsed_url.username, mask, 1)
+    if parsed_url.password:
+        proxy_url = proxy_url.replace(parsed_url.password, mask, 1)
+    return proxy_url
+
+
 class ProxyConfiguration(object):
     """Represents a proxy configuration dictionary and additional settings.
 
@@ -413,7 +431,7 @@ def send(self, request):
         except (NewConnectionError, socket.gaierror) as e:
             raise EndpointConnectionError(endpoint_url=request.url, error=e)
         except ProxyError as e:
-            raise ProxyConnectionError(proxy_url=proxy_url, error=e)
+            raise ProxyConnectionError(proxy_url=mask_proxy_url(proxy_url), error=e)
         except URLLib3ConnectTimeoutError as e:
             raise ConnectTimeoutError(endpoint_url=request.url, error=e)
         except URLLib3ReadTimeoutError as e:

docs/source/conf.py

@@ -54,7 +54,7 @@
 # The short X.Y version.
 version = '1.23.'
 # The full version, including alpha/beta/rc tags.
-release = '1.23.40'
+release = '1.23.41'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

tests/unit/test_http_session.py

@@ -1,15 +1,30 @@
 import socket
 
 import pytest
-from urllib3.exceptions import NewConnectionError, ProtocolError
+from urllib3.exceptions import (
+    NewConnectionError,
+    ProtocolError,
+    ProxyError,
+)
 
 from tests import mock, unittest
 
-from botocore.awsrequest import AWSRequest
-from botocore.awsrequest import AWSHTTPConnectionPool, AWSHTTPSConnectionPool
-from botocore.httpsession import get_cert_path
-from botocore.httpsession import URLLib3Session, ProxyConfiguration
-from botocore.exceptions import ConnectionClosedError, EndpointConnectionError
+from botocore.awsrequest import (
+    AWSRequest,
+    AWSHTTPConnectionPool,
+    AWSHTTPSConnectionPool,
+)
+from botocore.httpsession import (
+    get_cert_path,
+    mask_proxy_url,
+    URLLib3Session,
+    ProxyConfiguration,
+)
+from botocore.exceptions import (
+    ConnectionClosedError,
+    EndpointConnectionError,
+    ProxyConnectionError,
+)
 
 
 class TestProxyConfiguration(unittest.TestCase):
@@ -63,6 +78,56 @@ def test_get_cert_path_certifi_or_default(self):
             self.assertEqual(path, cert_path)
 
 
+@pytest.mark.parametrize(
+    'proxy_url, expected_mask_url',
+    (
+
+        (
+            'http://myproxy.amazonaws.com',
+            'http://myproxy.amazonaws.com'
+        ),
+        (
+            'http://user@myproxy.amazonaws.com',
+            'http://***@myproxy.amazonaws.com'
+        ),
+        (
+            'http://user:pass@myproxy.amazonaws.com',
+            'http://***:***@myproxy.amazonaws.com'
+        ),
+        (
+            'https://user:pass@myproxy.amazonaws.com',
+            'https://***:***@myproxy.amazonaws.com'
+        ),
+        (
+            'http://user:pass@localhost',
+            'http://***:***@localhost'
+        ),
+        (
+            'http://user:pass@localhost:80',
+            'http://***:***@localhost:80'
+        ),
+        (
+            'http://user:pass@userpass.com',
+            'http://***:***@userpass.com'
+        ),
+        (
+            'http://user:pass@192.168.1.1',
+            'http://***:***@192.168.1.1'
+        ),
+        (
+            'http://user:pass@[::1]',
+            'http://***:***@[::1]'
+        ),
+        (
+            'http://user:pass@[::1]:80',
+            'http://***:***@[::1]:80'
+        ),
+    )
+)
+def test_mask_proxy_url(proxy_url, expected_mask_url):
+    assert mask_proxy_url(proxy_url) == expected_mask_url
+
+
 class TestURLLib3Session(unittest.TestCase):
     def setUp(self):
         self.request = AWSRequest(
@@ -399,6 +464,14 @@ def test_catches_bad_status_line(self):
         with pytest.raises(ConnectionClosedError):
             self.make_request_with_error(error)
 
+    def test_catches_proxy_error(self):
+        self.connection.urlopen.side_effect = ProxyError('test', None)
+        session = URLLib3Session(proxies={'http': 'http://user:pass@proxy.com'})
+        with pytest.raises(ProxyConnectionError) as e:
+            session.send(self.request.prepare())
+        assert 'user:pass' not in str(e.value)
+        assert 'http://***:***@proxy.com' in str(e.value)
+
     def test_aws_connection_classes_are_used(self):
         session = URLLib3Session() # noqa
         # ensure the pool manager is using the correct classes