Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Prevent validator errors/retries on read timeouts #1056

Merged
merged 4 commits into from
Jan 26, 2022
Merged

Prevent validator errors/retries on read timeouts #1056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
df7e94c
Prevent retries on read_timeouts
ml-evs Jan 26, 2022
9136b72
Increase default read timeout to 60 seconds
ml-evs Jan 26, 2022
ef650dc
Update optimade/validator/utils.py
ml-evs Jan 26, 2022
0653af2
Merge branch 'master' into ml-evs/validator_tweaks
ml-evs Jan 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 9 additions & 1 deletion optimade/validator/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# pylint: disable=import-outside-toplevel
from optimade import __version__, __api_version__
from .validator import ImplementationValidator
from .utils import DEFAULT_CONN_TIMEOUT
from .utils import DEFAULT_CONN_TIMEOUT, DEFAULT_READ_TIMEOUT

__all__ = ["ImplementationValidator", "validate"]

Expand Down Expand Up @@ -112,6 +112,13 @@ def validate(): # pragma: no cover
help=f"Timeout to use for each individual request (DEFAULT: {DEFAULT_CONN_TIMEOUT} s)",
)

parser.add_argument(
"--read-timeout",
type=float,
default=DEFAULT_READ_TIMEOUT,
help=f"Read timeout to use for each individual request (DEFAULT: {DEFAULT_READ_TIMEOUT} s)",
)

args = vars(parser.parse_args())

if os.environ.get("OPTIMADE_VERBOSITY") is not None:
Expand Down Expand Up @@ -145,6 +152,7 @@ def validate(): # pragma: no cover
page_limit=args["page_limit"],
http_headers=args["headers"],
timeout=args["timeout"],
read_timeout=args["read_timeout"],
)

try:
Expand Down
6 changes: 5 additions & 1 deletion optimade/validator/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
# Default connection timeout allows for one default-sized TCP retransmission window
# (see https://docs.python-requests.org/en/latest/user/advanced/#timeouts)
DEFAULT_CONN_TIMEOUT = 3.05
DEFAULT_READ_TIMEOUT = 30
DEFAULT_READ_TIMEOUT = 60


class ResponseError(Exception):
Expand Down Expand Up @@ -247,6 +247,10 @@ def get(self, request: str):
except requests.exceptions.ConnectionError as exc:
errors.append(str(exc))

# Read timeouts should prevent further retries
except requests.exceptions.ReadTimeout as exc:
raise ResponseError(str(exc)) from exc

except requests.exceptions.MissingSchema:
sys.exit(
f"Unable to make request on {self.last_request}, did you mean http://{self.last_request}?"
Expand Down