Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 39 additions & 27 deletions easypost/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,31 @@ def convert_to_easypost_object(response, api_key, parent=None, name=None):
return response


def _utf8(value):
if six.PY2:
# Python2's urlencode wants bytestrings, not unicode
if isinstance(value, six.text_type):
return value.encode('utf-8')
return value
elif isinstance(value, six.binary_type):
# Python3's six.text_type(bytestring) returns "b'bytestring'"
# So, have to decode it to unicode
return value.decode('utf-8')
else:
# Python3's urlencode can handle unicode
return value


def _urlencode_list(params):
encoded = []
for key, values in sorted(params.items()):
for value in values:
if isinstance(value, bool):
value = str(value).lower()
encoded.append('{0}[]={1}'.format(key, quote_plus(_utf8(value))))
return '&'.join(encoded)


class Requestor(object):
def __init__(self, local_api_key=None):
self._api_key = local_api_key
Expand All @@ -162,35 +187,20 @@ def api_url(cls, url=None):
url = url or ''
return '%s%s' % (api_base, url)

@classmethod
def _utf8(cls, value):
if six.PY2:
# Python2's urlencode wants bytestrings, not unicode
if isinstance(value, six.text_type):
return value.encode('utf-8')
return value
elif isinstance(value, six.binary_type):
# Python3's six.text_type(bytestring) returns "b'bytestring'"
# So, have to decode it to unicode
return value.decode('utf-8')
else:
# Python3's urlencode can handle unicode
return value

@classmethod
def encode_dict(cls, out, key, dict_value):
n = {}
for k, v in sorted(six.iteritems(dict_value)):
k = cls._utf8(k)
v = cls._utf8(v)
k = _utf8(k)
v = _utf8(v)
n["%s[%s]" % (key, k)] = v
out.extend(cls._encode_inner(n))

@classmethod
def encode_list(cls, out, key, list_value):
n = {}
for k, v in enumerate(list_value):
v = cls._utf8(v)
v = _utf8(v)
n["%s[%s]" % (key, k)] = v
out.extend(cls._encode_inner(n))

Expand Down Expand Up @@ -218,7 +228,7 @@ def _encode_inner(cls, params):

out = []
for key, value in sorted(six.iteritems(params)):
key = cls._utf8(key)
key = _utf8(key)
try:
encoder = ENCODERS[value.__class__]
encoder(out, key, value)
Expand Down Expand Up @@ -575,7 +585,7 @@ def instance_url(self):
easypost_id = self.get('id')
if not easypost_id:
raise Error('%s instance has invalid ID: %r' % (type(self).__name__, easypost_id))
easypost_id = Requestor._utf8(easypost_id)
easypost_id = _utf8(easypost_id)
base = self.class_url()
param = quote_plus(easypost_id)
return "{base}/{param}".format(base=base, param=param)
Expand Down Expand Up @@ -635,13 +645,15 @@ def create(cls, api_key=None, verify=None, verify_strict=None, **params):
requestor = Requestor(api_key)
url = cls.class_url()

if verify or verify_strict:
verify = verify or []
verify_strict = verify_strict or []
url += '?' + '&'.join(
['verify[]={0}'.format(opt) for opt in verify] +
['verify_strict[]={0}'.format(opt) for opt in verify_strict]
)
verify_params = {}
for key, value in (('verify', verify), ('verify_strict', verify_strict)):
if not value:
continue
elif isinstance(value, (bool, str)):
value = [value]
verify_params[key] = value
if verify_params:
url += '?' + _urlencode_list(verify_params)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not clear to me why this list encoding needs its own special function, as opposed to passing the list as a parameter to requestor.request. It looks like _urlencode_list behaves slightly differently than the default list encoder in that it doesn't include an index inside the [], and our API docs do not indicate that an index should be included in the param name. Perhaps the other list encoder is incorrect?

Copy link
Contributor Author

@Roguelazer Roguelazer Jun 3, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

frankly I think we should just switch to encoding the bodies to JSON and get rid of all of this (absolutely awful) urlencoding crap

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that we should eventually move away from the nasty URL encoding for URL params. I looked into it some more, and part of the challenge is that our underlying HTTP library has to support both Python Requests (the good one), and also Google App Engine's urlfetcher (which is somewhat non-standard, and probably enforced for performance/security reasons on their cloud compute environment).

The way the client library is structured now has it so that URL parameters are encoded earlier on, before we get to the part where the HTTP library is even engaged.

Python Requests has a clean interface to encode URL params that also takes care of the URL encoding issues, but we aren't taking advantage of this.

In a future iteration of this library, I'd like to see the URL encoding logic pushed as far down as possible, and maintain a Python dictionary for the URL parameters and payload throughout most of the code, and do the necessary conversions only where necessary.

I would like to see the "multiple-path handling" logic for accommodating the two different HTTP libraries eventually be implemented as two different adapters or strategies hidden behind a wrapper for cleaner abstraction.

  1. For the second part, regarding lists of params, I had a hard time finding an official spec, but this SO details some of the issues.

To pass in a list of params, the two common practices are to do one of:

a) base/url?fav_foods=pizza&fav_foods=ice_cream&fav_foods=sushi OR
b) base/url?fav_foods[]=pizza&fav_foods[]=ice_cream&fav_foods[]=sushi

It's very wrong to have a list index inside the param. The former works, but various HTTP frameworks aren't implemented correctly to handle it, and the latter is also not idea, being a pattern that is used by some popular HTTP frameworks like Django and others, but again, not ubiquitous.

frankly I think we should just switch to encoding the bodies to JSON and get rid of all of this

James's suggestion to do away with URL parameters and just encode the whole thing and pass as a JSON body is the most correct/ideal.

In Conclusion

Given all of the options (and the difficulty in implementing them right now), and that this change is correct, I am comfortable accepting and merging this code as is.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I strongly agree with moving away from URL-encoding request parameters.

@jontsai putting an adapter in front of the HTTP request logic is a good idea, whether or not we stick with URL encoding parameters. In addition to that, I would love to drop support for urlfetch, as it's not in the GAE Python 3 runtime. Sadly, however, Google has stated that they will continue to support the 2.7 runtime indefinitely, so it's not clear when we'll be able to do that.


wrapped_params = {cls.class_name(): params}
response, api_key = requestor.request('post', url, wrapped_params)
Expand Down
83 changes: 83 additions & 0 deletions tests/cassettes/test_address_creation_with_verify_bool.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
interactions:
- request:
body: address%5Bcity%5D=San+Francisco&address%5Bcompany%5D=EasyPost&address%5Bcountry%5D=US&address%5Bphone%5D=415-456-7890&address%5Bstate%5D=CA&address%5Bstreet1%5D=118+2&address%5Bstreet2%5D=FLoor+4&address%5Bzip%5D=94105
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Content-Length:
- '218'
Content-type:
- application/x-www-form-urlencoded
authorization:
- EZTK-NONE
user-agent:
- easypost/v2 pythonclient/suppressed
x-client-user-agent:
- suppressed
method: POST
uri: https://api.easypost.com/v2/addresses?verify%5B%5D=true
response:
body:
string: !!binary |
H4sIAAAAAAAAA4ySzU7jMBSFXyXylhRsN2nS7KIySCOhgkhZjEYocuwbxiPHrmwHARXvznULs5hV
s3Hu8Xeuj38ORCvSEKF8v+IDH0ZVg2R1IYWoGZMg1uuCVeuqHgaSEzf8BRmRb5XyEAJK0oOIoHqR
ZE45W9DVgrId503JG0YvKG0oRXDeq/NAKyYgjZ2Nwe5u2gv7ho4fbffr/q7bIRCiB4gMRcbqjG+v
s26X3dxmxb85jnMpm47J2rXb7Oah3W5+dpu7I4NBUN+0WLzrPf7iJmm5WK54CiDdbKNPzscOy/0f
ZxNesLIoV1W9TgxMQpvvlJNTCYgQYrIL7zX4fhRSm2OCE4UHphXYqAUaR2EC5GQEBV6YPorXPl3E
iTwG/E97Aa9HLUXUzgbSHFLwIo1hljJdRRP9jB3Be+ex+v2UEwURU4ZTi49UG4193s61HYjB9eKc
dresLqu6qsqcGGefv8QF4/xyiU+kyknUE/Tvp6NqJ0wrxdWtC31rn8FAIB/4fQIAAP//AwDow9Tp
bgIAAA==
headers:
cache-control:
- no-cache, no-store
content-encoding:
- gzip
content-type:
- application/json; charset=utf-8
etag:
- W/"b7d8f69ebe4ba9aee3182e54d399f72a"
expires:
- '0'
location:
- /api/v2/addresses/adr_62b2bfd8ec184caa811cea99417978bb
pragma:
- no-cache
referrer-policy:
- strict-origin-when-cross-origin
strict-transport-security:
- max-age=15768000; includeSubDomains; preload
transfer-encoding:
- chunked
x-backend:
- easypost
x-content-type-options:
- nosniff
x-download-options:
- noopen
x-ep-request-uuid:
- d6502afc60b6ba1afb32a2c100f6fc6a
x-frame-options:
- SAMEORIGIN
x-node:
- bigweb5nuq
x-permitted-cross-domain-policies:
- none
x-proxied:
- intlb1nuq 15c8815ace
- extlb1nuq 15c8815ace
x-request-id:
- 1d5229af-27c8-4ffb-a2f9-b2fa88d35b9e
x-runtime:
- '0.056680'
x-version-label:
- easypost-202106012134-e838b9cad4-master
x-xss-protection:
- 1; mode=block
status:
code: 201
message: Created
version: 1
21 changes: 21 additions & 0 deletions tests/test_address.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,27 @@ def test_address_creation_with_verify():
assert address.country == 'US'


@pytest.mark.vcr()
def test_address_creation_with_verify_bool():
# Create an address with a verify parameter to test that it verifies accurately
address = easypost.Address.create(
verify=True,
street1='118 2',
street2='FLoor 4',
city='San Francisco',
state='CA',
zip='94105',
country='US',
company='EasyPost',
phone='415-456-7890'
)

assert address.id is not None
assert address.street1 == '118 2ND ST FL 4'
assert address.street2 == ''
assert address.country == 'US'


@pytest.mark.vcr()
def test_address_creation_with_verify_failure():
# Create an address with a verify parameter to test that it fails elegantly
Expand Down