Merge pull request #4907 from rcarpa/patch-4906-improve_disk_vs_tape_…

…priority Transfers: reduce priority of tape sources. #4906
rucio · Oct 19, 2021 · d36d346 · d36d346
2 parents de42772 + e7eb6e7
commit d36d346
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 3 deletions.
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
@@ -1235,12 +1235,15 @@ def __compress_multihops(
 def __sort_paths(candidate_paths: "Iterable[List[DirectTransferDefinition]]") -> "Generator[List[DirectTransferDefinition]]":
 
     def __transfer_order_key(transfer_path):
+        # Reduce the priority of the tape sources. If there are any disk sources,
+        # they must fail twice (1 penalty + 1 disk preferred over tape) before a tape will even be tried
+        source_ranking_penalty = 1 if transfer_path[0].src.rse.is_tape_or_staging_required() else 0
         # higher source_ranking first,
         # on equal source_ranking, prefer DISK over TAPE
         # on equal type, prefer lower distance_ranking
         # on equal distance, prefer single hop
         return (
-            - transfer_path[0].src.source_ranking,
+            - transfer_path[0].src.source_ranking + source_ranking_penalty,
             transfer_path[0].src.rse.is_tape_or_staging_required(),  # rely on the fact that False < True
             transfer_path[0].src.distance_ranking,
             len(transfer_path) > 1,  # rely on the fact that False < True

diff --git a/lib/rucio/tests/test_transfer.py b/lib/rucio/tests/test_transfer.py
@@ -184,10 +184,17 @@ def __fake_source_ranking(source_rse_id, new_ranking, session=None):
     assert len(transfer[0].legacy_sources) == 2
     assert transfer[0].legacy_sources[0][0] in (disk1_rse_name, disk2_rse_name)
 
-    # Change the rating of the disk RSEs. Tape RSEs must now be preferred.
-    # Multiple tape sources are not allowed. Only one tape RSE source must be returned.
+    # Change the rating of the disk RSEs. Disk still preferred, because it must fail twice before tape is tried
     __fake_source_ranking(disk1_rse_id, -1)
     __fake_source_ranking(disk2_rse_id, -1)
+    [[_host, [transfer]]] = next_transfers_to_submit(rses=all_rses).items()
+    assert len(transfer[0].legacy_sources) == 2
+    assert transfer[0].legacy_sources[0][0] in (disk1_rse_name, disk2_rse_name)
+
+    # Change the rating of the disk RSEs again. Tape RSEs must now be preferred.
+    # Multiple tape sources are not allowed. Only one tape RSE source must be returned.
+    __fake_source_ranking(disk1_rse_id, -2)
+    __fake_source_ranking(disk2_rse_id, -2)
     [[_host, transfers]] = next_transfers_to_submit(rses=all_rses).items()
     assert len(transfers) == 1
     transfer = transfers[0]