Skip to content

Commit

Permalink
Remove urllib3 as a dependency downloading preloads (#5199)
Browse files Browse the repository at this point in the history
We accidentally introduced urllib3 as a dependency in #5194. This PR fixes that mistake.
  • Loading branch information
marcosmoyano committed Aug 11, 2021
1 parent ac55f25 commit cbe97d1
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 12 deletions.
9 changes: 4 additions & 5 deletions distributed/preloading.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
import os
import shutil
import sys
import urllib.request
from importlib import import_module
from types import ModuleType
from typing import List

import click
import urllib3

from dask.utils import tmpfile

Expand Down Expand Up @@ -123,10 +123,9 @@ def _download_module(url: str) -> ModuleType:
logger.info("Downloading preload at %s", url)
assert is_webaddress(url)

client = urllib3.PoolManager()

response = client.request("GET", url)
source = response.data.decode()
request = urllib.request.Request(url, method="GET")
response = urllib.request.urlopen(request)
source = response.read().decode()

compiled = compile(source, url, "exec")
module = ModuleType(url)
Expand Down
15 changes: 8 additions & 7 deletions distributed/tests/test_preload.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
import sys
import tempfile
import time
import urllib.error
import urllib.request

import pytest
import tornado
import urllib3
from tornado import web

import dask
Expand Down Expand Up @@ -184,14 +185,14 @@ def scheduler_preload():
raise AssertionError("Process didn't come up")
time.sleep(0.5)
# Make sure we can query the server
client = urllib3.PoolManager()
start = time.time()
request = urllib.request.Request("http://127.0.0.1:12345/preload", method="GET")
while True:
try:
response = client.request("GET", "http://127.0.0.1:12345/preload")
response = urllib.request.urlopen(request)
if response.status == 200:
break
except urllib3.exceptions.HTTPError as e:
except urllib.error.URLError as e:
if time.time() > start + 10:
raise AssertionError("Webserver didn't come up", e)
time.sleep(0.5)
Expand Down Expand Up @@ -261,14 +262,14 @@ def worker_preload():
raise AssertionError("Process didn't come up")
time.sleep(0.5)
# Make sure we can query the server
client = urllib3.PoolManager()
request = urllib.request.Request("http://127.0.0.1:12346/preload", method="GET")
start = time.time()
while True:
try:
response = client.request("GET", "http://127.0.0.1:12346/preload")
response = urllib.request.urlopen(request)
if response.status == 200:
break
except urllib3.exceptions.HTTPError as e:
except urllib.error.URLError as e:
if time.time() > start + 10:
raise AssertionError("Webserver didn't come up", e)
time.sleep(0.5)
Expand Down

0 comments on commit cbe97d1

Please sign in to comment.