Fix resetting metadata on many repositories at once via the shed API

lib/tool_shed/test/base/populators.py

@@ -1,3 +1,6 @@
+import tarfile
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import (
     List,
     Optional,
@@ -7,6 +10,7 @@
 import requests
 
 from galaxy.util.resources import (
+    files,
     resource_path,
     Traversable,
 )
@@ -43,6 +47,28 @@
 COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
 COLUMN_MAKER_1_1_1_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
 DEFAULT_COMMIT_MESSAGE = "a test commit message"
+TEST_DATA_REPO_FILES = files("tool_shed.test.test_data")
+
+
+def repo_files(test_data_path: str) -> List[Path]:
+    repos = TEST_DATA_REPO_FILES.joinpath(f"repos/{test_data_path}")
+    paths = sorted(Path(str(x)) for x in repos.iterdir())
+    return paths
+
+
+def repo_tars(test_data_path: str) -> List[Path]:
+    tar_paths = []
+    for path in repo_files(test_data_path):
+        if path.is_dir():
+            prefix = f"shedtest_{test_data_path}_{path.name}_"
+            tf = NamedTemporaryFile(delete=False, prefix=prefix)
+            with tarfile.open(tf.name, "w:gz") as tar:
+                tar.add(str(path.absolute()), arcname=test_data_path or path.name)
+            tar_path = tf.name
+        else:
+            tar_path = str(path)
+        tar_paths.append(Path(tar_path))
+    return tar_paths
 
 
 class ToolShedPopulator:
@@ -55,6 +81,23 @@ def __init__(self, admin_api_interactor: ShedApiInteractor, api_interactor: Shed
         self._admin_api_interactor = admin_api_interactor
         self._api_interactor = api_interactor
 
+    def setup_test_data_repo(self, test_data_path: str) -> Repository:
+        prefix = test_data_path.replace("_", "")
+        category_id = self.new_category(prefix=prefix).id
+        repository = self.new_repository(category_id, prefix=prefix)
+        repository_id = repository.id
+        assert repository_id
+
+        for index, repo_tar in enumerate(repo_tars(test_data_path)):
+            commit_message = f"Updating {test_data_path} with index {index} with tar {repo_tar}"
+            response = self.upload_revision(
+                repository_id,
+                repo_tar,
+                commit_message=commit_message,
+            )
+            assert response.is_ok
+        return repository
+
     def setup_column_maker_repo(self, prefix=DEFAULT_PREFIX) -> Repository:
         category_id = self.new_category(prefix=prefix).id
         repository = self.new_repository(category_id, prefix=prefix)
@@ -110,7 +153,19 @@ def upload_revision(
         self, repository: HasRepositoryId, path: Traversable, commit_message: str = DEFAULT_COMMIT_MESSAGE
     ):
         response = self.upload_revision_raw(repository, path, commit_message)
-        api_asserts.assert_status_code_is_ok(response)
+        if response.status_code != 200:
+            response_json = None
+            err_msg = None
+            try:
+                response_json = response.json()
+            except Exception:
+                pass
+            if response_json and "err_msg" in response_json:
+                err_msg = response_json["err_msg"]
+            if err_msg and "No changes" in err_msg:
+                assert_msg = f"Updating repository [{repository}] with path [{path}] and commit_message {commit_message} failed to update repository contents, no changes found. Response: [{response_json}]"
+                raise AssertionError(assert_msg)
+            api_asserts.assert_status_code_is_ok(response)
         return RepositoryUpdate(__root__=response.json())
 
     def new_repository(self, category_id, prefix=DEFAULT_PREFIX) -> Repository:
@@ -169,6 +224,11 @@ def get_ordered_installable_revisions(self, owner: str, name: str) -> OrderedIns
         api_asserts.assert_status_code_is_ok(revisions_response)
         return OrderedInstallableRevisions(__root__=revisions_response.json())
 
+    def assert_has_n_installable_revisions(self, repository: Repository, n: int):
+        revisions = self.get_ordered_installable_revisions(repository.owner, repository.name)
+        actual_n = len(revisions.__root__)
+        assert actual_n == n, f"Expected {n} repository revisions, found {actual_n} for {repository}"
+
     def get_repository_for(self, owner: str, name: str, deleted: str = "false") -> Optional[Repository]:
         request = RepositoryIndexRequest(
             owner=owner,

lib/tool_shed/test/functional/test_shed_repositories.py

@@ -1,5 +1,11 @@
+import os
+import tempfile
+
+from galaxy.tool_util.parser import get_tool_source
+from galaxy.util.compression_utils import CompressedFile
 from galaxy.util.resources import resource_path
 from galaxy_test.base import api_asserts
+from tool_shed.test.base.populators import repo_tars
 from ..base.api import ShedApiTestCase
 
 COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
@@ -101,3 +107,52 @@ def test_repository_search(self):
         first_hit = results.hits[0]
         assert first_hit.repository.name == repository.name
         assert first_hit.repository.times_downloaded == 0
+
+    def test_repo_tars(self):
+        for index, repo_path in enumerate(repo_tars("column_maker")):
+            path = CompressedFile(repo_path).extract(tempfile.mkdtemp())
+            tool_xml_path = os.path.join(path, "column_maker.xml")
+            tool_source = get_tool_source(config_file=tool_xml_path)
+            tool_version = tool_source.parse_version()
+            if index == 0:
+                assert tool_version == "1.1.0"
+            elif index == 1:
+                assert tool_version == "1.2.0"
+            elif index == 2:
+                assert tool_version == "1.3.0"
+            else:
+                raise AssertionError("Wrong number of repo tars returned...")
+
+    def test_reset_on_simple_repository(self):
+        populator = self.populator
+        repository = populator.setup_test_data_repo("column_maker")
+        populator.assert_has_n_installable_revisions(repository, 3)
+        response = self.api_interactor.post(
+            "repositories/reset_metadata_on_repository", data={"repository_id": repository.id}
+        )
+        api_asserts.assert_status_code_is_ok(response)
+        populator.assert_has_n_installable_revisions(repository, 3)
+
+    def test_reset_with_uninstallable_revisions(self):
+        populator = self.populator
+        # setup a repository with 4 revisions but only 3 installable ones due to no version change in a tool
+        repository = populator.setup_test_data_repo("column_maker_with_download_gaps")
+        populator.assert_has_n_installable_revisions(repository, 3)
+        response = self.api_interactor.post(
+            "repositories/reset_metadata_on_repository", data={"repository_id": repository.id}
+        )
+        api_asserts.assert_status_code_is_ok(response)
+        populator.assert_has_n_installable_revisions(repository, 3)
+
+    def test_reset_all(self):
+        populator = self.populator
+        repository = populator.setup_test_data_repo("column_maker_with_download_gaps")
+        populator.assert_has_n_installable_revisions(repository, 3)
+        # reseting one at a time or resetting everything via the web controllers works...
+        # reseting all at once via the API does not work - it breaks the repository
+        response = self.api_interactor.post(
+            "repositories/reset_metadata_on_repositories",
+            data={"payload": "can not be empty because bug in controller"},
+        )
+        api_asserts.assert_status_code_is_ok(response)
+        populator.assert_has_n_installable_revisions(repository, 3)

lib/tool_shed/test/test_data/repos/column_maker/1/column_maker.xml

@@ -0,0 +1,83 @@
+<tool id="Add_a_column1" name="Compute" version="1.1.0">
+  <description>an expression on every row</description>
+  <command interpreter="python">
+    column_maker.py $input $out_file1 "$cond" $round ${input.metadata.columns} "${input.metadata.column_types}"
+  </command>
+  <inputs>
+    <param name="cond" size="40" type="text" value="c3-c2" label="Add expression"/>
+    <param format="tabular" name="input" type="data" label="as a new column to" help="Query missing? See TIP below"/>
+    <param name="round" type="select" label="Round result?">
+      <option value="no">NO</option>
+      <option value="yes">YES</option>
+    </param>    
+  </inputs>
+  <outputs>
+    <data format="input" name="out_file1" metadata_source="input"/>
+  </outputs>
+  <tests>
+    <test>
+      <param name="cond" value="c3-c2"/>
+      <param name="input" value="1.bed"/>
+      <param name="round" value="no"/>
+      <output name="out_file1" file="column_maker_out1.interval"/>
+    </test>
+    <test>
+      <param name="cond" value="c4*1"/>
+      <param name="input" value="1.interval"/>
+      <param name="round" value="no"/>
+      <output name="out_file1" file="column_maker_out2.interval"/>
+    </test>
+    <test>
+      <param name="cond" value="c4*1"/>
+      <param name="input" value="1.interval"/>
+      <param name="round" value="yes"/>
+      <output name="out_file1" file="column_maker_out3.interval"/>
+    </test>
+  </tests>
+  <help>
+
+ .. class:: infomark
+
+**TIP:** If your data is not TAB delimited, use *Text Manipulation-&gt;Convert*
+
+-----
+
+**What it does**
+
+This tool computes an expression for every row of a query and appends the result as a new column (field).
+
+- Columns are referenced with **c** and a **number**. For example, **c1** refers to the first column of a tab-delimited file
+
+- **c3-c2** will add a length column to the query if **c2** and **c3** are start and end position
+
+-----
+
+**Example**
+
+If this is your input::
+
+   chr1  151077881  151077918  2  200  -
+   chr1  151081985  151082078  3  500  +
+
+computing "c4*c5" will produce::
+
+   chr1  151077881  151077918  2  200  -   400.0
+   chr1  151081985  151082078  3  500  +  1500.0
+
+if, at the same time, "Round result?" is set to **YES** results will look like this::
+
+   chr1  151077881  151077918  2  200  -   400
+   chr1  151081985  151082078  3  500  +  1500
+
+You can also use this tool to evaluate expressions. For example, computing "c3>=c2" for Input will result in the following::
+
+   chr1  151077881  151077918  2  200  -  True
+   chr1  151081985  151082078  3  500  +  True
+
+or computing "type(c2)==type('') for Input will return::
+
+   chr1  151077881  151077918  2  200  -  False
+   chr1  151081985  151082078  3  500  +  False
+
+</help>
+</tool>

lib/tool_shed/test/test_data/repos/column_maker/2/column_maker.xml

@@ -0,0 +1,83 @@
+<tool id="Add_a_column1" name="Compute" version="1.2.0">
+  <description>an expression on every row</description>
+  <command interpreter="python">
+    column_maker.py $input $out_file1 "$cond" $round ${input.metadata.columns} "${input.metadata.column_types}"
+  </command>
+  <inputs>
+    <param name="cond" size="50" type="text" value="c3-c2" label="Add expression"/>
+    <param format="tabular" name="input" type="data" label="as a new column to" help="Query missing? See TIP below"/>
+    <param name="round" type="select" label="Round result?">
+      <option value="no">NO</option>
+      <option value="yes">YES</option>
+    </param>    
+  </inputs>
+  <outputs>
+    <data format="input" name="out_file1" metadata_source="input"/>
+  </outputs>
+  <tests>
+    <test>
+      <param name="cond" value="c3-c2"/>
+      <param name="input" value="1.bed"/>
+      <param name="round" value="no"/>
+      <output name="out_file1" file="column_maker_out1.interval"/>
+    </test>
+    <test>
+      <param name="cond" value="c4*1"/>
+      <param name="input" value="1.interval"/>
+      <param name="round" value="no"/>
+      <output name="out_file1" file="column_maker_out2.interval"/>
+    </test>
+    <test>
+      <param name="cond" value="c4*1"/>
+      <param name="input" value="1.interval"/>
+      <param name="round" value="yes"/>
+      <output name="out_file1" file="column_maker_out3.interval"/>
+    </test>
+  </tests>
+  <help>
+
+ .. class:: infomark
+
+**TIP:** If your data is not TAB delimited, use *Text Manipulation-&gt;Convert*
+
+-----
+
+**What it does**
+
+This tool computes an expression for every row of a query and appends the result as a new column (field).
+
+- Columns are referenced with **c** and a **number**. For example, **c1** refers to the first column of a tab-delimited file
+
+- **c3-c2** will add a length column to the query if **c2** and **c3** are start and end position
+
+-----
+
+**Example**
+
+If this is your input::
+
+   chr1  151077881  151077918  2  200  -
+   chr1  151081985  151082078  3  500  +
+
+computing "c4*c5" will produce::
+
+   chr1  151077881  151077918  2  200  -   400.0
+   chr1  151081985  151082078  3  500  +  1500.0
+
+if, at the same time, "Round result?" is set to **YES** results will look like this::
+
+   chr1  151077881  151077918  2  200  -   400
+   chr1  151081985  151082078  3  500  +  1500
+
+You can also use this tool to evaluate expressions. For example, computing "c3>=c2" for Input will result in the following::
+
+   chr1  151077881  151077918  2  200  -  True
+   chr1  151081985  151082078  3  500  +  True
+
+or computing "type(c2)==type('') for Input will return::
+
+   chr1  151077881  151077918  2  200  -  False
+   chr1  151081985  151082078  3  500  +  False
+
+</help>
+</tool>