Merge dc1b610 into d0f8b62

tests/ubiconfig/loaders/test_gitlab_loader.py

@@ -11,6 +11,12 @@ def mock_json(value):
     return out
 
 
+def mock_header():
+    out = MagicMock()
+    out.headers = {'Content-Length': '629', 'X-Total-Pages': '1', 'X-Per-Page': '20'}
+    return out
+
+
 def test_bad_yaml():
     with patch('requests.Session') as mock_session_class:
         session = mock_session_class.return_value
@@ -19,14 +25,17 @@ def test_bad_yaml():
             mock_json([{'name': 'master',
                         'commit': {'id': '2189cbc2e447f796fe354f8d784d76b0a2620248'}}]),
 
+            # headers
+            mock_header(),
+
             # files
             mock_json([{'name': 'badfile.yaml', 'path': 'badfile.yaml'}]),
 
             # content (not valid yaml!)
             Mock(content='[oops not yaml'),
         ]
 
-        loader = GitlabLoader('https://some-repo.example.com/foo/bar', per_page=30)
+        loader = GitlabLoader('https://some-repo.example.com/foo/bar')
 
         # It should propagate the YAML load exception
         with pytest.raises(yaml.YAMLError):

tests/ubiconfig/test_ubi.py

@@ -144,6 +144,8 @@ def test_get_branches(mocked_session):
                  'commit': {'id': 'c99cb8d7dae2e78e8cc7e720d3f950d1c5a0b51f'}},
                 {'name': 'ubi7',
                  'commit': {'id': '2189cbc2e447f796fe354f8d784d76b0a2620248'}}]
+    headers = {'Content-Length': '629', 'X-Total-Pages': '1', 'X-Per-Page': '20'}
+    mocked_session.return_value.get.return_value.headers = headers
     mocked_session.return_value.get.return_value.json.return_value = branches
     loader = ubi.get_loader()
     actual_branches = loader._get_branches()
@@ -157,6 +159,8 @@ def test_pre_load(mocked_get_branches, mocked_session, files_branch_map):
     branches = ['c99cb8d7dae2e78e8cc7e720d3f950d1c5a0b51f',
                 '2189cbc2e447f796fe354f8d784d76b0a2620248']
     mocked_get_branches.return_value = branches
+    headers = {'Content-Length': '629', 'X-Total-Pages': '1', 'X-Per-Page': '20'}
+    mocked_session.return_value.get.return_value.headers = headers
     file_list = [[{'name': 'rhel-atomic-host.yaml', 'path': 'rhel-atomic-host.yaml'},
                   {'name': 'README.md', 'path': 'README.md'}],
                  [{'name': 'rhel-7-for-power-le.yaml', 'path': 'rhel-7-for-power-le.yaml'}]]

tests/ubiconfig/utils/test_gitlab_api.py

@@ -24,8 +24,8 @@ def test_get_branch_list_api(v4_repo_api, v4_api_prefix):
 
 
 def test_get_file_list_api(v4_repo_api, v4_api_prefix):
-    expected_file_list_api = urljoin(v4_api_prefix, 'repository/tree?ref=master&per_page=30')
-    v4_repo_api.get_file_list_api() == expected_file_list_api
+    expected_file_list_api = urljoin(v4_api_prefix, 'repository/tree?ref=master&page=1')
+    v4_repo_api.get_file_list_api(page=1) == expected_file_list_api
 
 
 def test_file_content_api(v4_repo_api, v4_api_prefix):

ubiconfig/_impl/loaders/gitlab.py

@@ -13,11 +13,11 @@
 
 class GitlabLoader(Loader):
     """Load configuration from a remote repo on gitlab."""
-    def __init__(self, url, per_page):
+    def __init__(self, url):
         self._url = url
         self._session = requests.Session()
         self._repo_api = RepoApi(self._url.rstrip('/'))
-        self._files_branch_map = self._pre_load(per_page)
+        self._files_branch_map = self._pre_load()
 
     def load(self, file_name):
         # find the right branch from the mapping
@@ -47,20 +47,24 @@ def load_all(self, recursive=False):
 
         return ubi_configs
 
-    def _pre_load(self, per_page):
+    def _pre_load(self):
         """Iterate all branches to get a mapping of {file_path: branch,...}
         """
         files_branch_map = {}
         branches = self._get_branches()
         LOG.info("Loading config files from all branches: %s", branches)
         for branch in branches:
-            file_list_api = self._repo_api.get_file_list_api(branch=branch,
-                                                             per_page=per_page)
-            json_response = self._session.get(file_list_api).json()
-            file_list = [file['path'] for file in json_response
-                         if file['name'].endswith(('.yaml', '.yml'))]
-            for file in file_list:
-                files_branch_map[file] = branch
+            headers_api = self._repo_api.get_file_list_api(branch=branch, page=1)
+            headers = self._session.get(headers_api).headers
+            total_pages = int(headers['X-Total-Pages'])
+            for page in range(1, total_pages+1):
+                file_list_api = self._repo_api.get_file_list_api(branch=branch,
+                                                                 page=page)
+                json_response = self._session.get(file_list_api).json()
+                file_list = [file['path'] for file in json_response
+                             if file['name'].endswith(('.yaml', '.yml'))]
+                for file in file_list:
+                    files_branch_map[file] = branch
         return files_branch_map
 
     def _get_branches(self):

ubiconfig/ubi.py

@@ -15,7 +15,7 @@ class LoaderError(RuntimeError):
     pass
 
 
-def get_loader(source=None, per_page=30):
+def get_loader(source=None):
     """Get a Loader instance which is used to load configurations.
 
     ``source`` should be provided as one of the following:
@@ -32,10 +32,6 @@ def get_loader(source=None, per_page=30):
             environment variable is used. If this is unset, an
             exception is raised.
 
-    ``per_page`` specifies the maximum number of files returned by load_all(),
-    and it's defaulted to 30.
-    It's only functional when load from remote repo
-
     After the loader is constructed, it can be used to load config files
     when given relative paths to config files.
 
@@ -62,7 +58,7 @@ def get_loader(source=None, per_page=30):
     parsed = urlparse(source)
     if parsed.netloc:
         # It's a URL, use the gitlab loader
-        return GitlabLoader(source, per_page)
+        return GitlabLoader(source)
 
     # It should be a local path
     if not os.path.isdir(source):

ubiconfig/utils/api/gitlab.py

@@ -38,13 +38,13 @@ def api_url(self):
     def get_branch_list_api(self):
         return urljoin(self.api_url, 'repository/branches')
 
-    def get_file_list_api(self, branch=None, per_page=30):
+    def get_file_list_api(self, page, branch=None):
         """Return the api used to get the list of files in the repo or files
         in the sub-module.
         The defualt maximum number of return files is 30
         """
         branch = branch.replace('/', '%2F') if branch else 'master'
-        return urljoin(self.api_url, 'repository/tree?ref=%s&per_page=%s' % (branch, per_page))
+        return urljoin(self.api_url, 'repository/tree?ref=%s&page=%s' % (branch, page))
 
     def get_file_content_api(self, file_path, branch=None):
         """Get the api used to retrieve the raw content.