Core & Internals: handle root proxy for geoip sorting. Closes #6242

When proxy servers are used the distance must be computed towards
the inner hostname, because the outer one is the proxy address and
will be the same for all replicas.

lib/rucio/core/replica_sorter.py

@@ -238,7 +238,13 @@ def sort_geoip(dictreplica: "Dict", client_location: "Dict", ignore_error: bool
     """
 
     def distance(pfn):
-        return __get_distance(urlparse(pfn).hostname, client_location, ignore_error)
+        url = urlparse(pfn)
+        if url.scheme == 'root':
+            # handle root proxy urls: root://10.0.0.1//root://192.168.1.1:1094//dpm/....
+            sub_url = urlparse(url.path.lstrip('/'))
+            if sub_url.scheme and sub_url.hostname:
+                url = sub_url
+        return __get_distance(url.hostname, client_location, ignore_error)
 
     return list(sorted(dictreplica, key=distance))
 

lib/rucio/tests/test_replica_sorting.py

@@ -25,6 +25,7 @@
 
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import parse_replicas_from_string
+from rucio.common.config import config_get
 from rucio.core import rse_expression_parser, replica_sorter
 from rucio.core.replica import add_replicas, delete_replicas
 from rucio.core.rse import add_rse, del_rse, add_rse_attribute, add_protocol, del_rse_attribute
@@ -46,6 +47,9 @@
     'Libya': '2a02:e700::1',
 }
 
+CLIENT_SITE = 'CLIENTSITE'
+CLIENT_SITE_CACHE = '10.0.0.1:443'
+
 base_rse_info = [
     {'site': 'APERTURE', 'address': 'aperture.com', 'ip': LOCATION_TO_IP['Austria']},
     {'site': 'BLACKMESA', 'address': 'blackmesa.com', 'ip': LOCATION_TO_IP['Japan']},
@@ -68,6 +72,12 @@ def mock_geoip_db():
         os.unlink(temp_db_path)
 
 
+@pytest.fixture
+def mock_get_multi_cache_prefix():
+    with mock.patch('rucio.core.replica.get_multi_cache_prefix', side_effect=lambda _x, _y: CLIENT_SITE_CACHE):
+        yield
+
+
 @pytest.fixture
 def mock_get_lat_long():
     def _get_lat_long_mock(se, gi):
@@ -172,14 +182,24 @@ def protocols_setup(vo):
 
 @pytest.mark.noparallel(reason='fails when run in parallel, lists replicas and checks for length of returned list')
 @pytest.mark.parametrize("content_type", [Mime.METALINK, Mime.JSON_STREAM])
-def test_sort_geoip_wan_client_location(vo, rest_client, auth_token, protocols_setup, content_type, mock_geoip_db, mock_get_lat_long):
+@pytest.mark.parametrize("file_config_mock", [
+    # Run test twice: with root proxy and without
+    {"overrides": []},
+    {"overrides": [('clientcachemap', CLIENT_SITE, 'ANYTHING')]},
+], indirect=True)
+def test_sort_geoip_wan_client_location(vo, rest_client, auth_token, protocols_setup, content_type,
+                                        mock_geoip_db, mock_get_lat_long, mock_get_multi_cache_prefix, file_config_mock):
     """Replicas: test sorting a few WANs via geoip."""
 
     data = {
         'dids': [{'scope': f['scope'].external, 'name': f['name'], 'type': 'FILE'} for f in protocols_setup['files']],
         'schemes': schemes,
         'sort': 'geoip',
     }
+    test_with_cache = False
+    if config_get('clientcachemap', CLIENT_SITE, raise_exception=False) is not None:
+        test_with_cache = True
+        data['client_location'] = {'site': CLIENT_SITE}
 
     first_aut_then_jpn = ['root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com', 'gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com']
     first_jpn_then_aut = ['gsiftp.blackmesa.com', 'davs.blackmesa.com', 'root.blackmesa.com', 'root.aperture.com', 'davs.aperture.com', 'gsiftp.aperture.com']
@@ -217,7 +237,15 @@ def test_sort_geoip_wan_client_location(vo, rest_client, auth_token, protocols_s
 
         print(client_location, pfns)
         assert len(replicas) == 1
-        assert [urlparse(pfn).hostname for pfn in pfns] == expected_order
+        if test_with_cache:
+            cache_prefix = f'root://{CLIENT_SITE_CACHE}//'
+            for i, pfn in enumerate(pfns):
+                if pfn.startswith('root'):
+                    assert pfn.startswith(cache_prefix)
+                    pfn = pfn[len(cache_prefix):]
+                assert urlparse(pfn).hostname == expected_order[i]
+        else:
+            assert [urlparse(pfn).hostname for pfn in pfns] == expected_order
 
 
 @pytest.mark.noparallel(reason='fails when run in parallel, lists replicas and checks for length of returned list')