Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support GET request method for simple API calls (not just file content) #474

Merged
merged 3 commits into from
Feb 8, 2024
Merged

support GET request method for simple API calls (not just file content) #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
281f346
add `GET` method support to B2Http
mjurbanski-reef Feb 7, 2024
197180a
remove `b2sdk.b2http:test_http` deadcode
mjurbanski-reef Feb 7, 2024
206da64
fix docstrings and type hints in b2http
mjurbanski-reef Feb 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions b2sdk/_v3/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,3 +260,6 @@

# filter
from b2sdk.filter import FilterType, Filter

# typing
from b2sdk.utils.typing import JSON
259 changes: 130 additions & 129 deletions b2sdk/b2http.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@
import time
from contextlib import contextmanager
from random import random
from typing import Any
from typing import Any, Callable

try:
from typing_extensions import Literal
except ImportError:
from typing import Literal

import requests
from requests.adapters import HTTPAdapter
Expand All @@ -40,6 +45,7 @@
interpret_b2_error,
)
from .requests import NotDecompressingResponse
from .utils.typing import JSON
from .version import USER_AGENT

LOCALE_LOCK = threading.Lock()
Expand Down Expand Up @@ -215,54 +221,101 @@ def add_callback(self, callback):
"""
self.callbacks.append(callback)

def post_content_return_json(
def request(
self,
url,
headers,
data,
method: Literal['POST', 'GET', 'HEAD'],
url: str,
headers: dict[str, str],
data: io.BytesIO | bytes | None = None,
try_count: int = TRY_COUNT_DATA,
post_params=None,
params: dict[str, str] | None = None,
*,
stream: bool = False,
_timeout: int | None = None,
):
) -> requests.Response:
"""
Use like this:

.. code-block:: python

try:
response_dict = b2_http.post_content_return_json(url, headers, data)
response_dict = b2_http.request('POST', url, headers, data)
...
except B2Error as e:
...

:param str url: a URL to call
:param dict headers: headers to send.
:param data: bytes (Python 3) or str (Python 2), or a file-like object, to send
:return: a dict that is the decoded JSON
:rtype: dict
:param method: uppercase HTTP method name
:param url: a URL to call
:param headers: headers to send.
:param data: raw bytes or a file-like object to send
:param try_count: a number of retries
:param params: a dict that will be converted to query string for GET requests or additional metadata for POST requests
:param stream: if True, the response will be streamed
:param _timeout: a timeout for the request in seconds if not default
:return: final response
:raises: B2Error if the request fails
"""
method = method.upper()
request_headers = {**headers, 'User-Agent': self.user_agent}

# Do the HTTP POST. This may retry, so each post needs to
# rewind the data back to the beginning.
def do_post():
data.seek(0)
self._run_pre_request_hooks('POST', url, request_headers)
response = self.session.post(
def do_request():
# This may retry, so each time we need to rewind the data back to the beginning.
if data is not None and not isinstance(data, bytes):
data.seek(0)
self._run_pre_request_hooks(method, url, request_headers)
response = self.session.request(
method,
url,
headers=request_headers,
data=data,
params=params if method == 'GET' else None,
timeout=(self.CONNECTION_TIMEOUT, _timeout or self.TIMEOUT_FOR_UPLOAD),
stream=stream,
)
self._run_post_request_hooks('POST', url, request_headers, response)
self._run_post_request_hooks(method, url, request_headers, response)
return response

try:
response = self._translate_and_retry(do_post, try_count, post_params)
except B2RequestTimeout:
# this forces a token refresh, which is necessary if request is still alive
# on the server but has terminated for some reason on the client. See #79
raise B2RequestTimeoutDuringUpload()
return self._translate_and_retry(do_request, try_count, params)

def request_content_return_json(
self,
method: Literal['POST', 'GET', 'HEAD'],
url: str,
headers: dict[str, str],
data: io.BytesIO | bytes | None = None,
try_count: int = TRY_COUNT_DATA,
params: dict[str, str] | None = None,
*,
_timeout: int | None = None,
) -> JSON:
"""
Use like this:

.. code-block:: python

try:
response_dict = b2_http.request_content_return_json('POST', url, headers, data)
...
except B2Error as e:
...

:param method: uppercase HTTP method name
:param url: a URL to call
:param headers: headers to send.
:param data: raw bytes or a file-like object to send
:return: decoded JSON
"""
response = self.request(
method,
url,
headers={
**headers, 'Accept': 'application/json'
},
data=data,
try_count=try_count,
params=params,
_timeout=_timeout
)

# Decode the JSON that came back. If we've gotten this far,
# we know we have a status of 200 OK. In this case, the body
Expand All @@ -273,6 +326,40 @@ def do_post():
finally:
response.close()

def post_content_return_json(
self,
url: str,
headers: dict[str, str],
data: bytes | io.IOBase,
try_count: int = TRY_COUNT_DATA,
post_params: dict[str, str] | None = None,
_timeout: int | None = None,
) -> JSON:
"""
Use like this:

.. code-block:: python

try:
response_dict = b2_http.post_content_return_json(url, headers, data)
...
except B2Error as e:
...

:param str url: a URL to call
:param dict headers: headers to send.
:param data: a file-like object to send
:return: a dict that is the decoded JSON
"""
try:
return self.request_content_return_json(
'POST', url, headers, data, try_count, post_params, _timeout=_timeout
)
except B2RequestTimeout:
# this forces a token refresh, which is necessary if request is still alive
# on the server but has terminated for some reason on the client. See #79
raise B2RequestTimeoutDuringUpload()

def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT_OTHER):
"""
Use like this:
Expand All @@ -298,10 +385,13 @@ def post_json_return_json(self, url, headers, params, try_count: int = TRY_COUNT
# to indicate the timeouts should be designed.
timeout = self.TIMEOUT_FOR_COPY

data = io.BytesIO(json.dumps(params).encode())
data = json.dumps(params).encode()
return self.post_content_return_json(
url,
headers,
{
**headers,
'Content-Type': 'application/json',
},
data,
try_count,
params,
Expand Down Expand Up @@ -332,29 +422,17 @@ def get_content(self, url, headers, try_count: int = TRY_COUNT_DOWNLOAD):
:param int try_count: a number or retries
:return: Context manager that returns an object that supports iter_content()
"""
request_headers = {**headers, 'User-Agent': self.user_agent}

# Do the HTTP GET.
def do_get():
self._run_pre_request_hooks('GET', url, request_headers)
response = self.session.get(
url,
headers=request_headers,
stream=True,
timeout=(self.CONNECTION_TIMEOUT, self.TIMEOUT),
)
self._run_post_request_hooks('GET', url, request_headers, response)
return response

response = self._translate_and_retry(do_get, try_count, None)
response = self.request(
'GET', url, headers=headers, try_count=try_count, stream=True, _timeout=self.TIMEOUT
)
return ResponseContextManager(response)

def head_content(
self,
url: str,
headers: dict[str, Any],
try_count: int = TRY_COUNT_HEAD,
) -> dict[str, Any]:
) -> requests.Response:
"""
Does a HEAD instead of a GET for the URL.
The response's content is limited to the headers.
Expand All @@ -375,24 +453,9 @@ def head_content(
:param str url: a URL to call
:param dict headers: headers to send
:param int try_count: a number or retries
:return: the decoded response
:rtype: dict
:return: HTTP response
"""
request_headers = {**headers, 'User-Agent': self.user_agent}

# Do the HTTP HEAD.
def do_head():
self._run_pre_request_hooks('HEAD', url, request_headers)
response = self.session.head(
url,
headers=request_headers,
stream=True,
timeout=(self.CONNECTION_TIMEOUT, self.TIMEOUT),
)
self._run_post_request_hooks('HEAD', url, request_headers, response)
return response

return self._translate_and_retry(do_head, try_count, None)
return self.request('HEAD', url, headers=headers, try_count=try_count)

@classmethod
def _get_user_agent(cls, user_agent_append):
Expand Down Expand Up @@ -505,13 +568,16 @@ def _translate_errors(cls, fcn, post_params=None):
raise UnknownError(text)

@classmethod
def _translate_and_retry(cls, fcn, try_count, post_params=None):
def _translate_and_retry(
cls, fcn: Callable, try_count: int, post_params: dict[str, Any] | None = None
):
"""
Try calling fcn try_count times, retrying only if
the exception is a retryable B2Error.

:param int try_count: a number of retries
:param dict post_params: request parameters
:param fcn: request function to call
:param try_count: a number of retries
:param post_params: request parameters
"""
# For all but the last try, catch the exception.
wait_time = 1.0
Expand Down Expand Up @@ -556,68 +622,3 @@ class NotDecompressingHTTPAdapter(HTTPAdapter):

def build_response(self, req, resp):
return NotDecompressingResponse.from_builtin_response(super().build_response(req, resp))


def test_http():
"""
Run a few tests on error diagnosis.

This test takes a while to run and is not used in the automated tests
during building. Run the test by hand to exercise the code.
"""

from .exception import BadJson

b2_http = B2Http()

# Error from B2
print('TEST: error object from B2')
try:
b2_http.post_json_return_json(
'https://api.backblazeb2.com/b2api/v1/b2_get_file_info', {}, {}
)
assert False, 'should have failed with bad json'
except BadJson as e:
assert str(e) == 'Bad request: required field fileId is missing'

# Successful get
print('TEST: get')
with b2_http.get_content(
'https://api.backblazeb2.com/test/echo_zeros?length=10', {}
) as response:
assert response.status_code == 200
response_data = b''.join(response.iter_content())
assert response_data == b'\x00' * 10

# Successful post
print('TEST: post')
response_dict = b2_http.post_json_return_json(
'https://api.backblazeb2.com/api/build_version', {}, {}
)
assert 'timestamp' in response_dict

# Unknown host
print('TEST: unknown host')
try:
b2_http.post_json_return_json('https://unknown.backblazeb2.com', {}, {})
assert False, 'should have failed with unknown host'
except UnknownHost:
pass

# Broken pipe
print('TEST: broken pipe')
try:
data = io.BytesIO(b'\x00' * 10000000)
b2_http.post_content_return_json('https://api.backblazeb2.com/bad_url', {}, data)
assert False, 'should have failed with broken pipe'
except BrokenPipe:
pass

# Generic connection error
print('TEST: generic connection error')
try:
with b2_http.get_content('https://www.backblazeb2.com:80/bad_url', {}) as response:
assert False, 'should have failed with connection error'
response.iter_content() # make pyflakes happy
except B2ConnectionError:
pass
Loading
Loading