Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
fix: optionally keep user-provided base URL for pagination (#2149)
  • Loading branch information
iomarmochtar committed Aug 3, 2022
1 parent 1136b17 commit e2ea8b8
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 0 deletions.
29 changes: 29 additions & 0 deletions gitlab/client.py
Expand Up @@ -17,6 +17,7 @@
"""Wrapper for the GitLab API."""

import os
import re
import time
from typing import Any, cast, Dict, List, Optional, Tuple, TYPE_CHECKING, Union

Expand Down Expand Up @@ -65,6 +66,8 @@ class Gitlab:
user_agent: A custom user agent to use for making HTTP requests.
retry_transient_errors: Whether to retry after 500, 502, 503, 504
or 52x responses. Defaults to False.
keep_base_url: keep user-provided base URL for pagination if it
differs from response headers
"""

def __init__(
Expand All @@ -84,6 +87,7 @@ def __init__(
order_by: Optional[str] = None,
user_agent: str = gitlab.const.USER_AGENT,
retry_transient_errors: bool = False,
keep_base_url: bool = False,
) -> None:

self._api_version = str(api_version)
Expand All @@ -94,6 +98,7 @@ def __init__(
#: Timeout to use for requests to gitlab server
self.timeout = timeout
self.retry_transient_errors = retry_transient_errors
self.keep_base_url = keep_base_url
#: Headers that will be used in request to GitLab
self.headers = {"User-Agent": user_agent}

Expand Down Expand Up @@ -1132,6 +1137,30 @@ def _query(
next_url = requests.utils.parse_header_links(result.headers["links"])[
0
]["url"]
# if the next url is different with user provided server URL
# then give a warning it may because of misconfiguration
# but if the option to fix provided then just reconstruct it
if not next_url.startswith(self._gl.url):
search_api_url = re.search(r"(^.*?/api)", next_url)
if search_api_url:
next_api_url = search_api_url.group(1)
if self._gl.keep_base_url:
next_url = next_url.replace(
next_api_url, f"{self._gl._base_url}/api"
)
else:
utils.warn(
message=(
f"The base URL in the server response"
f"differs from the user-provided base URL "
f"({self._gl.url}/api/ -> {next_api_url}/). "
f"This may lead to unexpected behavior and "
f"broken pagination. Use `keep_base_url=True` "
f"when initializing the Gitlab instance "
f"to follow the user-provided base URL."
),
category=UserWarning,
)
self._next_url = next_url
except KeyError:
self._next_url = None
Expand Down
55 changes: 55 additions & 0 deletions tests/unit/test_gitlab.py
Expand Up @@ -353,3 +353,58 @@ def test_gitlab_plain_const_does_not_warn(recwarn):

assert not recwarn
assert no_access == 0


@responses.activate
@pytest.mark.parametrize(
"kwargs,link_header,expected_next_url,show_warning",
[
(
{},
"<http://localhost/api/v4/tests?per_page=1&page=2>;" ' rel="next"',
"http://localhost/api/v4/tests?per_page=1&page=2",
False,
),
(
{},
"<http://orig_host/api/v4/tests?per_page=1&page=2>;" ' rel="next"',
"http://orig_host/api/v4/tests?per_page=1&page=2",
True,
),
(
{"keep_base_url": True},
"<http://orig_host/api/v4/tests?per_page=1&page=2>;" ' rel="next"',
"http://localhost/api/v4/tests?per_page=1&page=2",
False,
),
],
ids=["url-match-does-not-warn", "url-mismatch-warns", "url-mismatch-keeps-url"],
)
def test_gitlab_keep_base_url(kwargs, link_header, expected_next_url, show_warning):
responses.add(
**{
"method": responses.GET,
"url": "http://localhost/api/v4/tests",
"json": [{"a": "b"}],
"headers": {
"X-Page": "1",
"X-Next-Page": "2",
"X-Per-Page": "1",
"X-Total-Pages": "2",
"X-Total": "2",
"Link": (link_header),
},
"content_type": "application/json",
"status": 200,
"match": helpers.MATCH_EMPTY_QUERY_PARAMS,
}
)

gl = gitlab.Gitlab(url="http://localhost", **kwargs)
if show_warning:
with pytest.warns(UserWarning) as warn_record:
obj = gl.http_list("/tests", iterator=True)
assert len(warn_record) == 1
else:
obj = gl.http_list("/tests", iterator=True)
assert obj._next_url == expected_next_url

0 comments on commit e2ea8b8

Please sign in to comment.