Skip to content

Commit

Permalink
Adds Range header support to content app
Browse files Browse the repository at this point in the history
This unskips the `Range` header and adds support for it to the content
app.

closes #8865
  • Loading branch information
bmbouter committed Jul 8, 2021
1 parent 9c1f115 commit 68de39f
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 7 deletions.
2 changes: 2 additions & 0 deletions CHANGES/8865.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixed bug where content app would not respond to `Range` HTTP Header in requests, e.g. from Anaconda
clients.
28 changes: 24 additions & 4 deletions pulpcore/content/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,8 @@ async def _match_and_stream(self, path, request):
:class:`aiohttp.web.StreamResponse` or :class:`aiohttp.web.FileResponse`: The response
streamed back to the client.
"""
# import pydevd_pycharm
# pydevd_pycharm.settrace('localhost', port=29437, stdoutToServer=True, stderrToServer=True)

def match_distribution_blocking():
return self._match_distribution(path)
Expand Down Expand Up @@ -665,7 +667,7 @@ def get_remote_artifacts_blocking():
return response

except (ClientResponseError, UnsupportedDigestValidationError) as e:
log.warn(
log.warning(
_("Could not download remote artifact at '{}': {}").format(
remote_artifact.url, str(e)
)
Expand Down Expand Up @@ -809,15 +811,33 @@ def cast_remote_blocking():

remote = await loop.run_in_executor(None, cast_remote_blocking)

async def handle_headers(headers):
rng = request.http_range

async def handle_response_headers(headers):
for name, value in headers.items():
if name.lower() in self.hop_by_hop_headers:
continue
response.headers[name] = value
await response.prepare(request)

async def handle_data(data):
await response.write(data)
if rng.start or rng.stop:
start_byte_pos = 0
end_byte_pos = len(data)
if rng.start:
start_byte_pos = max(0, rng.start - downloader._size)
if rng.stop:
end_byte_pos = min(len(data), rng.stop - downloader._size)

if start_byte_pos < 0:
raise Exception("start is less than zerooooo")
if end_byte_pos > len(data):
raise Exception("end is more than len(data)")

data_for_client = data[start_byte_pos:end_byte_pos]
await response.write(data_for_client)
else:
await response.write(data)
if remote.policy != Remote.STREAMED:
await original_handle_data(data)

Expand All @@ -826,7 +846,7 @@ async def finalize():
await original_finalize()

downloader = remote.get_downloader(
remote_artifact=remote_artifact, headers_ready_callback=handle_headers
remote_artifact=remote_artifact, headers_ready_callback=handle_response_headers
)
original_handle_data = downloader.handle_data
downloader.handle_data = handle_data
Expand Down
3 changes: 2 additions & 1 deletion pulpcore/download/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,8 @@ async def _handle_response(self, response):
if self.headers_ready_callback:
await self.headers_ready_callback(response.headers)
while True:
chunk = await response.content.read(1048576) # 1 megabyte
# chunk = await response.content.read(1048576) # 1 megabyte
chunk = await response.content.read(3) # 3 bytes
if not chunk:
await self.finalize()
break # the download is done
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,6 @@ def test_content_served_immediate(self):
self.setup_download_test("immediate")
self.do_test_content_served()

@unittest.skip("https://pulp.plan.io/issues/8865")
def test_content_served_on_demand_with_range_request(self):
"""Assert that on_demand content can be properly downloaded with range requests."""
self.setup_download_test("on_demand")
Expand Down Expand Up @@ -389,9 +388,11 @@ def do_test_content_served(self):
self.assertEqual(len(pulp_manifest), FILE_FIXTURE_COUNT, pulp_manifest)

def do_range_request_download_test(self):
# import pydevd_pycharm
# pydevd_pycharm.settrace('localhost', port=29437, stdoutToServer=True, stderrToServer=True)
file_path = "1.iso"

headers = {"Range": "bytes=0-9"} # first 10 bytes
headers = {"Range": "bytes=2-11"} # first 10 bytes
NUM_BYTES = 10

req1 = download_content_unit(
Expand Down

0 comments on commit 68de39f

Please sign in to comment.