Client : Bulk get_metadata method : Closes #3645

AUTHORS.rst

@@ -43,6 +43,7 @@ Individual contributors to the source code
 - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2019
 - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+- Muhammad Aditya Hilmy <didithilmy@gmail.com>, 2020
 
 Organisations employing contributors
 ------------------------------------
@@ -66,3 +67,4 @@ Organisations employing contributors
 - Dwarkadas J. Sanghvi College of Engineering (India)
 - Science and Technology Facilities Council (UK)
 - Brookhaven National Laboratory (USA)
+- Institut Teknologi Bandung (Indonesia)

doc/source/restapi/credentials.rst

@@ -0,0 +1,12 @@
+Credential Rest API
+======================
+
+**Overview**
+
+.. qrefflask:: rucio.web.rest.flaskapi.v1.credential:make_doc()
+     :undoc-static:
+
+**Details**
+
+.. autoflask:: rucio.web.rest.flaskapi.v1.credential:make_doc()
+     :undoc-static:

doc/source/restapi/exporter.rst

@@ -0,0 +1,12 @@
+Exporter Rest API
+======================
+
+**Overview**
+
+.. qrefflask:: rucio.web.rest.flaskapi.v1.exporter:make_doc()
+     :undoc-static:
+
+**Details**
+
+.. autoflask:: rucio.web.rest.flaskapi.v1.exporter:make_doc()
+     :undoc-static:

doc/source/restapi/importer.rst

@@ -0,0 +1,12 @@
+Importer Rest API
+======================
+
+**Overview**
+
+.. qrefflask:: rucio.web.rest.flaskapi.v1.importer:make_doc()
+     :undoc-static:
+
+**Details**
+
+.. autoflask:: rucio.web.rest.flaskapi.v1.importer:make_doc()
+     :undoc-static:

lib/rucio/core/replica.py

@@ -490,7 +490,7 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, session=None
                               models.RSEFileAssociation.rse_id).\
             filter(models.RSEFileAssociation.state == ReplicaState.BAD)
 
-    query = filter_thread_work(session=session, query=query, total_threads=total_threads, thread_id=thread, hash_variable='name')
+    query = filter_thread_work(session=session, query=query, total_threads=total_threads, thread_id=thread, hash_variable='%sreplicas.name' % (schema_dot))
     query = query.join(models.DataIdentifier,
                        and_(models.DataIdentifier.scope == models.RSEFileAssociation.scope,
                             models.DataIdentifier.name == models.RSEFileAssociation.name)).\

lib/rucio/core/rule.py

@@ -855,6 +855,7 @@ def list_associated_rules_for_file(scope, name, session=None):
     :raises:        RucioException
     """
 
+    rucio.core.did.get_did(scope=scope, name=name, session=session)  # Check if the did acually exists
     query = session.query(models.ReplicationRule).\
         with_hint(models.ReplicaLock, "INDEX(LOCKS LOCKS_PK)", 'oracle').\
         join(models.ReplicaLock, models.ReplicationRule.id == models.ReplicaLock.rule_id).\

lib/rucio/daemons/conveyor/submitter.py

@@ -363,7 +363,7 @@ def __sort_link_ranking(sources):
             rank_sources[link_ranking] = []
         rank_sources[link_ranking].append(source)
     rank_keys = list(rank_sources.keys())
-    rank_keys.sort(reverse=True)
+    rank_keys.sort()
     for rank_key in rank_keys:
         sources_list = rank_sources[rank_key]
         random.shuffle(sources_list)

lib/rucio/rse/protocols/xrootd.py

@@ -44,7 +44,7 @@ def path2pfn(self, path):
             :returns: Fully qualified PFN.
 
         """
-        if not path.startswith('xroot'):
+        if not path.startswith('xroot') and not path.startswith('root'):
             if path.startswith('/'):
                 return '%s://%s:%s/%s' % (self.scheme, self.hostname, self.port, path)
             else:
@@ -113,9 +113,22 @@ def stat(self, path):
         return ret
 
     def pfn2path(self, pfn):
-        parse_pfns = self.parse_pfns(pfn)[pfn]
-        path = parse_pfns['prefix'] + parse_pfns['path'] + parse_pfns['name']
-        return path
+        """
+        Returns the path of a file given the pfn, i.e. scheme and hostname are subtracted from the pfn.
+
+        :param path: pfn of a file
+
+        :returns: path.
+        """
+        if pfn.startswith('//'):
+            return pfn
+        elif pfn.startswith('/'):
+            return '/' + pfn
+        else:
+            prefix = self.attributes['prefix']
+            path = pfn.partition(self.attributes['prefix'])[2]
+            path = prefix + path
+            return path
 
     def lfns2pfns(self, lfns):
         """
@@ -152,7 +165,6 @@ def connect(self):
 
             :raises RSEAccessDenied
         """
-
         try:
             # The query stats call is not implemented on some xroot doors.
             # Workaround: fail, if server does not reply within 10 seconds for static config query
@@ -202,7 +214,6 @@ def put(self, filename, target, source_dir, transfer_timeout=None):
         """
         source_url = '%s/%s' % (source_dir, filename)
         path = self.path2pfn(target)
-
         if not os.path.exists(source_url):
             raise exception.SourceNotFound()
         try:

lib/rucio/web/rest/flaskapi/v1/account_limit.py

@@ -125,7 +125,7 @@ def delete(self, account, rse):
 def make_doc():
     """ Only used for sphinx documentation to add the prefix """
     doc_app = Flask(__name__)
-    doc_app.register_blueprint(bp, url_prefix='/accounts')
+    doc_app.register_blueprint(bp, url_prefix='/accountlimits')
     return doc_app
 
 

lib/rucio/web/rest/flaskapi/v1/config.py

@@ -185,7 +185,7 @@ def put(self, section, option, value):
 def make_doc():
     """ Only used for sphinx documentation to add the prefix """
     doc_app = Flask(__name__)
-    doc_app.register_blueprint(bp, url_prefix='/configs')
+    doc_app.register_blueprint(bp, url_prefix='/config')
     return doc_app
 
 

lib/rucio/web/rest/flaskapi/v1/exporter.py

@@ -69,7 +69,7 @@ def get(self):
 bp = Blueprint('export', __name__)
 
 export_view = Export.as_view('scope')
-bp.add_url_rule('', view_func=export_view, methods=['get', ])
+bp.add_url_rule('/', view_func=export_view, methods=['get', ])
 
 application = Flask(__name__)
 application.register_blueprint(bp)

lib/rucio/web/rest/flaskapi/v1/importer.py

@@ -77,7 +77,7 @@ def post(self):
 bp = Blueprint('import', __name__)
 
 import_view = Import.as_view('scope')
-bp.add_url_rule('', view_func=import_view, methods=['post', ])
+bp.add_url_rule('/', view_func=import_view, methods=['post', ])
 
 application = Flask(__name__)
 application.register_blueprint(bp)

lib/rucio/web/rest/flaskapi/v1/rule.py

@@ -557,7 +557,7 @@ def get(self, rule_id):
 def make_doc():
     """ Only used for sphinx documentation to add the prefix """
     doc_app = Flask(__name__)
-    doc_app.register_blueprint(bp, url_prefix='/rule')
+    doc_app.register_blueprint(bp, url_prefix='/rules')
     return doc_app
 
 

lib/rucio/web/rest/flaskapi/v1/temporary_did.py

@@ -87,7 +87,7 @@ def POST(self):
 def make_doc():
     """ Only used for sphinx documentation to add the prefix """
     doc_app = Flask(__name__)
-    doc_app.register_blueprint(bp, url_prefix='/temporary_dids')
+    doc_app.register_blueprint(bp, url_prefix='/tmp_dids')
     return doc_app
 
 

lib/rucio/web/rest/webpy/v1/did.py

@@ -20,7 +20,7 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2018
 # - Yun-Pin Sun <yun-pin.sun@cern.ch>, 2013
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2018
-# - Martin Baristis <martin.barisits@cern.ch>, 2014-2015
+# - Martin Baristis <martin.barisits@cern.ch>, 2014-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Ruturaj Gujar, <ruturaj.gujar23@gmail.com>, 2019
 #
@@ -703,6 +703,8 @@ def GET(self, scope, name):
         try:
             for rule in list_associated_replication_rules_for_file(scope=scope, name=name):
                 yield dumps(rule, cls=APIEncoder) + '\n'
+        except DataIdentifierNotFound as error:
+            raise generate_http_error(404, 'DataIdentifierNotFound', error.args[0])
         except RucioException as error:
             raise generate_http_error(500, error.__class__.__name__, error.args[0])
         except Exception as error: