Client : Add add-exception command in the CLI : Closes rucio#5292

bin/rucio

@@ -20,7 +20,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2012-2022
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2021
 # - Yun-Pin Sun <winter0128@gmail.com>, 2012-2013
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2022
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - David Cameron <david.cameron@cern.ch>, 2014-2021
 # - Tomáš Kouba <tomas.kouba@cern.ch>, 2014
@@ -1460,7 +1460,6 @@ def info_rule(args):
     if args.estimate_ttc:
         logger.error('"--estimate-ttc" is deprecated!')
         return FAILURE
-
     client = get_client(args)
     if args.examine:
         analysis = client.examine_replication_rule(rule_id=args.rule_id)
@@ -1503,7 +1502,6 @@ def info_rule(args):
         print("Notification:               %s" % rule['notification'])
         print("End of life:                %s" % rule['eol_at'])
         print("Child Rule Id:              %s" % rule['child_rule_id'])
-
     return SUCCESS
 
 
@@ -1796,6 +1794,91 @@ def list_datasets_rse(args):
     return SUCCESS
 
 
+@exception_handler
+def add_lifetime_exception(args):
+    """
+    %(prog)s add_lifetime_exception [options] <field1=value1 field2=value2 ...>
+
+    Declare a lifetime model exception.
+
+    """
+    client = get_client(args)
+    if not args.reason:
+        logger.error('reason for the extension is mandatory')
+        return FAILURE
+    reason = args.reason
+    if not args.expiration:
+        logger.error('expiration is mandatory')
+        return FAILURE
+    try:
+        expiration = datetime.strptime(args.expiration, "%Y-%m-%d")
+    except Exception as err:
+        logger.error(err)
+        return FAILURE
+
+    if not args.inputfile:
+        logger.error('inputfile is mandatory')
+        return FAILURE
+    with open(args.inputfile) as infile:
+        dids = list(filter(None, [line.strip() for line in infile]))
+
+    dids_list = []
+    containers = []
+    datasets = []
+    error_types = ['Total DIDs', 'DID not submitted because it is a file', 'DID not submitted because it is a container', 'DID not submitted because it is not part of the lifetime campaign', 'DID successfully submitted']
+    for did in dids:
+        scope, name = get_scope(did, client)
+        dids_list.append({'scope': scope, 'name': name})
+    summary = {0: len(dids_list), 1: 0, 2: 0, 3: 0, 4: 0}
+    for meta in client.get_metadata_bulk(dids_list):
+        scope, name = meta['scope'], meta['name']
+        dids_list.remove({'scope': scope, 'name': name})
+        if meta['did_type'] == 'FILE':
+            logger.warning('%s:%s is a file. Will be ignored' % (scope, name))
+            summary[1] += 1
+        elif meta['did_type'] == 'CONTAINER':
+            logger.warning('%s:%s is a container. It needs to be resolved' % (scope, name))
+            containers.append({'scope': scope, 'name': name})
+            summary[2] += 1
+        elif not meta['eol_at']:
+            logger.warning('%s:%s is not affected by the lifetime model' % (scope, name))
+            summary[3] += 1
+        else:
+            logger.info('%s:%s will be declared' % (scope, name))
+            datasets.append({'scope': scope, 'name': name})
+            summary[4] += 1
+
+    for did in dids_list:
+        scope = did['scope']
+        name = did['name']
+        logger.warning('%s:%s does not exist' % (scope, name))
+
+    if containers:
+        logger.error('One or more DIDs are containers. They must be resolved into a list of datasets to request exception. Full list below')
+        for container in containers:
+            logger.error('%s:%s' % (container['scope'], container['name']))
+        logger.error('Please take action. Submission aborted')
+        return FAILURE
+
+    if not datasets:
+        logger.error('Nothing to submit')
+        return SUCCESS
+    try:
+        client.add_exception(dids=datasets, account=client.account, pattern='', comments=reason, expires_at=expiration)
+    except UnsupportedOperation as err:
+        logger.error(err)
+        return FAILURE
+    except Exception:
+        logger.error('Failure to submit exception. Please retry')
+        logger.debug(traceback.format_exc())
+        return FAILURE
+
+    logger.info('Exception successfully submitted. Summary below')
+    for cnt, error in enumerate(error_types):
+        print('{0:100} {1:6d}'.format(error, len(summary[cnt])))
+    return SUCCESS
+
+
 def test_server(args):
     """"
     %(prog)s test-server [options] <field1=value1 field2=value2 ...>
@@ -2480,6 +2563,21 @@ can be found in ' + Color.BOLD + 'http://rucio.cern.ch/documentation/rse_express
     touch_parser.add_argument(dest='dids', nargs='+', action='store', help='List of space separated data identifiers.')
     touch_parser.add_argument('--rse', dest='rse', action='store', help="The RSE of the DIDs that are touched.").completer = rse_completer
 
+    # The add-lifetime-exception command
+    add_lifetime_exception_parser = subparsers.add_parser('add-lifetime-exception', help='Add an exception to the lifetime model.',
+                                                          formatter_class=argparse.RawDescriptionHelpFormatter, epilog='''Usage example
+"""""""""""""
+::
+
+    $ rucio add-lifetime-exception --inputfile myfile.txt --reason "Needed for my analysis" --expiration 2015-10-30
+
+    ''')
+
+    add_lifetime_exception_parser.set_defaults(function=add_lifetime_exception)
+    add_lifetime_exception_parser.add_argument('--inputfile', action='store', help='File where the list of datasets requested to be extended are located.', required=True)
+    add_lifetime_exception_parser.add_argument('--reason', action='store', help='The reason for the extension.'), required=True
+    add_lifetime_exception_parser.add_argument('--expiration', action='store', help='The expiration date format YYYY-MM-DD', required=True)
+
     return oparser
 
 

lib/rucio/client/lifetimeclient.py

@@ -1,4 +1,5 @@
-# Copyright 2017-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2017-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +14,13 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2017
-# - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2017-2022
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2018
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Igor Mandrichenko <igorvm@gmail.com>, 2021
 
 from __future__ import print_function
 
@@ -73,9 +77,7 @@ def add_exception(self, dids, account, pattern, comments, expires_at):
         path = self.LIFETIME_BASEURL + '/'
         url = build_url(choice(self.list_hosts), path=path)
         data = {'dids': dids, 'account': account, 'pattern': pattern, 'comments': comments, 'expires_at': expires_at}
-        print(render_json(**data))
         result = self._send_request(url, type_='POST', data=render_json(**data))
-        print(result.text)
         if result.status_code == codes.created:
             return loads(result.text)
         exc_cls, exc_msg = self._get_exception(headers=result.headers, status_code=result.status_code, data=result.content)

lib/rucio/tests/test_bin_rucio.py

@@ -20,7 +20,7 @@
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012-2021
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2014-2018
 # - Cheng-Hsi Chao <cheng-hsi.chao@cern.ch>, 2014
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2015-2022
 # - Martin Barisits <martin.barisits@cern.ch>, 2015-2022
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2017-2018
 # - Tobias Wegner <twegner@cern.ch>, 2018
@@ -76,8 +76,7 @@ def conf_vo(self):
                 # Client-only test, only use config with no DB config
                 self.vo = {'vo': get_long_vo()}
             try:
-                remove(get_tmp_dir()
-                       + '/.rucio_root@%s/auth_token_for_account_root' % self.vo['vo'])
+                remove(get_tmp_dir() + '/.rucio_root@%s/auth_token_for_account_root' % self.vo['vo'])
             except OSError as error:
                 if error.args[0] != 2:
                     raise error
@@ -2172,3 +2171,39 @@ def test_rucio_create_rule_with_0_copies(self):
         print(out)
         assert exitcode != 0
         assert "The number of copies for a replication rule should be greater than 0." in err
+
+    def test_add_lifetime_exception(self):
+        """ CLIENT(USER): Rucio submission of lifetime exception """
+        container = 'container_%s' % generate_uuid()
+        dataset = 'dataset_%s' % generate_uuid()
+        self.did_client.add_container(scope=self.user, name=container)
+        self.did_client.add_dataset(scope=self.user, name=dataset)
+        filename = get_tmp_dir() + 'lifetime_exception.txt'
+        with open(filename, 'w') as file_:
+            file_.write('%s:%s\n' % (self.user, dataset))
+
+        # Try adding an exception
+        cmd = 'rucio add-lifetime-exception --inputfile %s --reason "%s" --expiration %s' % (filename, 'Needed for analysis', '2015-10-30')
+        print(cmd)
+        exitcode, out, err = execute(cmd)
+        print(exitcode, out, err)
+        assert exitcode == 0
+        assert "Nothing to submit" in err
+
+        with open(filename, 'w') as file_:
+            file_.write('%s:%s\n' % (self.user, dataset))
+            file_.write('%s:%s' % (self.user, container))
+
+        # Try adding an exception
+        cmd = 'rucio add-lifetime-exception --inputfile %s --reason "%s" --expiration %s' % (filename, 'Needed for analysis', '2015-10-30')
+        print(cmd)
+        exitcode, out, err = execute(cmd)
+        print(exitcode, out, err)
+        assert exitcode == 1
+        assert "One or more DIDs are containers" in err
+
+        try:
+            remove(filename)
+        except OSError as e:
+            if e.args[0] != 2:
+                raise e