diff --git a/lib/common/interface/mysqlstore.py b/lib/common/interface/mysqlstore.py index 168f4349..a7690970 100644 --- a/lib/common/interface/mysqlstore.py +++ b/lib/common/interface/mysqlstore.py @@ -523,6 +523,14 @@ def _do_load_files(self, dataset): #override dataset.files.add(lfile) + def _do_check_if_on(self, datasetname, sitename): #override + query = 'SELECT COUNT(*) FROM `datasets` AS d' + query += ' INNER JOIN `dataset_replicas` AS dr ON dr.`dataset_id` = d.`id`' + query += ' INNER JOIN `sites` AS s ON s.`id` = dr.`site_id`' + query += ' WHERE d.`name` = %s and s.`name` = %s' + + return (self._mysql.query(query, datasetname, sitename)[0] != 0) + def _do_find_block_of(self, fullpath, datasets): #override query = 'SELECT d.`name`, b.`name` FROM `files` AS f' query += ' INNER JOIN `datasets` AS d ON d.`id` = f.`dataset_id`' diff --git a/lib/common/interface/store.py b/lib/common/interface/store.py index 6e3221f2..90e8751d 100644 --- a/lib/common/interface/store.py +++ b/lib/common/interface/store.py @@ -268,6 +268,15 @@ def load_files(self, dataset): finally: self.release_lock() + def check_if_on(self, datasetname, sitename): + """ + Return true/false if replica is on specific site. + """ + + logger.debug('_do_check_if_on()') + + return self._do_check_if_on(datasetname, sitename) + def find_block_of(self, fullpath, datasets): """ Return the Block object for the given file. diff --git a/lib/detox/configuration.py b/lib/detox/configuration.py index 10b1e1f1..843d65b1 100644 --- a/lib/detox/configuration.py +++ b/lib/detox/configuration.py @@ -4,5 +4,6 @@ activity_indicator = '/home/cmsprod/public_html/IntelROCCS/Detox/inActionLock.txt', deletion_per_iteration = 0.01, # fraction of quota to delete per iteration deletion_volume_per_request = 50, # size to delete per deletion request in TB + exclude_if_on = ['T1_IT_CNAF_MSS'], # if a dataset has a replica on these [sites], don't consider it for deletions. Introduced because of CNAF indicent. time_shift = 0. # number of days in the future from which to evaluate the policies ) diff --git a/lib/detox/main.py b/lib/detox/main.py index 5b260ab3..3c91b373 100644 --- a/lib/detox/main.py +++ b/lib/detox/main.py @@ -99,6 +99,16 @@ def _execute_policy(self, policy, is_test, comment): # will also select out replicas on sites with quotas all_replicas = policy.partition_replicas(self.inventory_manager, target_sites) + # check if replica is present on other site(s) that trigger exclusion from the possible deletion + # possible use case: a tape site has had a water indicident, for example + # Communication with database is needed because we do not have all (tape) replicas in memory + if len(detox_config.main.exclude_if_on) > 0: + for replica in all_replicas: + ds_name = replica.dataset.name + for sitename in detox_config.main.exclude_if_on: + if self.inventory_manager.store.check_if_on(ds_name,sitename): + replica.dataset.demand['on_protected_site'] = True + logger.info('Saving site and dataset states.') # update site and dataset lists diff --git a/lib/detox/variables.py b/lib/detox/variables.py index 7ae5fc95..4b107bdb 100644 --- a/lib/detox/variables.py +++ b/lib/detox/variables.py @@ -104,6 +104,16 @@ def _get(self, dataset): except KeyError: return 0. +class DatasetOnProtectedSite(DatasetAttr): + def __init__(self): + DatasetAttr.__init__(self, Attr.BOOL_TYPE) + + def _get(self, dataset): + try: + return dataset.demand['on_protected_site'] + except KeyError: + return False + class ReplicaSize(DatasetReplicaAttr): def __init__(self): DatasetReplicaAttr.__init__(self, Attr.NUMERIC_TYPE) @@ -272,6 +282,7 @@ def _get(self, site): 'dataset.usage_rank': DatasetUsageRank(), 'dataset.demand_rank': DatasetDemandRank(), 'dataset.release': DatasetRelease(), + 'dataset.on_protected_site': DatasetOnProtectedSite(), 'replica.is_last_transfer_source': ReplicaIsLastSource(), 'replica.size': ReplicaSize(), 'replica.incomplete': ReplicaIncomplete(),