From ae14ec78b27065bbc3213846e549778311056f34 Mon Sep 17 00:00:00 2001 From: Benedikt Maier Date: Tue, 21 Nov 2017 13:57:46 -0500 Subject: [PATCH 1/4] Not considering replicas for deletions if on certain site --- lib/common/interface/mysqlstore.py | 8 ++++++++ lib/common/interface/store.py | 9 +++++++++ lib/detox/configuration.py | 3 ++- lib/detox/main.py | 13 +++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/common/interface/mysqlstore.py b/lib/common/interface/mysqlstore.py index 168f4349..577b8b84 100644 --- a/lib/common/interface/mysqlstore.py +++ b/lib/common/interface/mysqlstore.py @@ -523,6 +523,14 @@ def _do_load_files(self, dataset): #override dataset.files.add(lfile) + def _do_check_if_on(self, datasetname, sitename): #override + query = 'SELECT 1 FROM `datasets` AS d' + query += ' INNER JOIN `dataset_replicas` AS dr ON dr.`dataset_id` = d.`id`' + query += ' INNER JOIN `sites` AS s ON s.`id` = dr.`site_id`' + query += ' WHERE d.`name` = %s and s.`name` = %s' % (datasetname, sitename) + + return self._mysql.query(query)[0] + def _do_find_block_of(self, fullpath, datasets): #override query = 'SELECT d.`name`, b.`name` FROM `files` AS f' query += ' INNER JOIN `datasets` AS d ON d.`id` = f.`dataset_id`' diff --git a/lib/common/interface/store.py b/lib/common/interface/store.py index 6e3221f2..90e8751d 100644 --- a/lib/common/interface/store.py +++ b/lib/common/interface/store.py @@ -268,6 +268,15 @@ def load_files(self, dataset): finally: self.release_lock() + def check_if_on(self, datasetname, sitename): + """ + Return true/false if replica is on specific site. + """ + + logger.debug('_do_check_if_on()') + + return self._do_check_if_on(datasetname, sitename) + def find_block_of(self, fullpath, datasets): """ Return the Block object for the given file. diff --git a/lib/detox/configuration.py b/lib/detox/configuration.py index 10b1e1f1..6aa0880b 100644 --- a/lib/detox/configuration.py +++ b/lib/detox/configuration.py @@ -2,7 +2,8 @@ main = Configuration( activity_indicator = '/home/cmsprod/public_html/IntelROCCS/Detox/inActionLock.txt', - deletion_per_iteration = 0.01, # fraction of quota to delete per iteration + deletion_per_iteration = 0, # fraction of quota to delete per iteration deletion_volume_per_request = 50, # size to delete per deletion request in TB + exclude_if_on = [], # if a dataset has a replica on these [sites], don't consider it for deletions. Introduced because of CNAF indicent. time_shift = 0. # number of days in the future from which to evaluate the policies ) diff --git a/lib/detox/main.py b/lib/detox/main.py index 5b260ab3..4dab2691 100644 --- a/lib/detox/main.py +++ b/lib/detox/main.py @@ -99,6 +99,19 @@ def _execute_policy(self, policy, is_test, comment): # will also select out replicas on sites with quotas all_replicas = policy.partition_replicas(self.inventory_manager, target_sites) + # check if replica is present on other site(s) that trigger exclusion from the possible deletion + # possible use case: a tape site has had a water indicident, for example + # Communication is needed because we do not + if len(detox_config.main.exclude_if_on) > 0: + excluded_replicas = [] + for replica in all_replicas: + ds_name = replica.dataset.name + for sitename in detox_config.main.exclude_if_on: + if self.inventory_manager.store.check_if_on(ds_name,sitename): + excluded_replicas.append(replica) + for excluded_replica in excluded_replicas: + all_replicas.remove(excluded_replica) + logger.info('Saving site and dataset states.') # update site and dataset lists From d2ec84a1769861e8e2c823be86846618454c0037 Mon Sep 17 00:00:00 2001 From: Benedikt Maier Date: Tue, 21 Nov 2017 14:03:30 -0500 Subject: [PATCH 2/4] Not considering replicas for deletions if on certain site --- lib/detox/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/detox/configuration.py b/lib/detox/configuration.py index 6aa0880b..b4b94389 100644 --- a/lib/detox/configuration.py +++ b/lib/detox/configuration.py @@ -2,7 +2,7 @@ main = Configuration( activity_indicator = '/home/cmsprod/public_html/IntelROCCS/Detox/inActionLock.txt', - deletion_per_iteration = 0, # fraction of quota to delete per iteration + deletion_per_iteration = 0.01, # fraction of quota to delete per iteration deletion_volume_per_request = 50, # size to delete per deletion request in TB exclude_if_on = [], # if a dataset has a replica on these [sites], don't consider it for deletions. Introduced because of CNAF indicent. time_shift = 0. # number of days in the future from which to evaluate the policies From 6fa5aaad1c42381efb7c204ee3e7bbbb370543ba Mon Sep 17 00:00:00 2001 From: Benedikt Maier Date: Tue, 21 Nov 2017 14:05:02 -0500 Subject: [PATCH 3/4] Not considering replicas for deletions if on certain site --- lib/detox/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/detox/main.py b/lib/detox/main.py index 4dab2691..1a4c240c 100644 --- a/lib/detox/main.py +++ b/lib/detox/main.py @@ -101,7 +101,7 @@ def _execute_policy(self, policy, is_test, comment): # check if replica is present on other site(s) that trigger exclusion from the possible deletion # possible use case: a tape site has had a water indicident, for example - # Communication is needed because we do not + # Communication with database is needed because we do not have all (tape) replicas in memory if len(detox_config.main.exclude_if_on) > 0: excluded_replicas = [] for replica in all_replicas: From 2e31f331057ccc94f22c04f4b2a5add2c5bc5025 Mon Sep 17 00:00:00 2001 From: Benedikt Maier Date: Fri, 24 Nov 2017 10:30:37 -0500 Subject: [PATCH 4/4] Having it as dataset variable --- lib/detox/variables.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/detox/variables.py b/lib/detox/variables.py index 7ae5fc95..3b388fe5 100644 --- a/lib/detox/variables.py +++ b/lib/detox/variables.py @@ -104,6 +104,16 @@ def _get(self, dataset): except KeyError: return 0. +class DatasetTapeSite(DatasetAttr): + def __init__(self): + DatasetAttr.__init__(self, Attr.NUMERIC_TYPE) + + def _get(self, dataset): + try: + return dataset.demand['archieved_at'] + except KeyError: + return "" + class ReplicaSize(DatasetReplicaAttr): def __init__(self): DatasetReplicaAttr.__init__(self, Attr.NUMERIC_TYPE) @@ -272,6 +282,7 @@ def _get(self, site): 'dataset.usage_rank': DatasetUsageRank(), 'dataset.demand_rank': DatasetDemandRank(), 'dataset.release': DatasetRelease(), + 'dataset.archieved_at' : DatasetTapeSite(), 'replica.is_last_transfer_source': ReplicaIsLastSource(), 'replica.size': ReplicaSize(), 'replica.incomplete': ReplicaIncomplete(),