diff --git a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py index 610bddd37e..ce16d01ace 100644 --- a/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py +++ b/Framework/script/RepoCleaner/qcrepocleaner/rules/1_per_run.py @@ -11,8 +11,9 @@ logger = logging # default logger -def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, extra_params: Dict[str, str]): - ''' +def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_timestamp: int, + extra_params: Dict[str, str]): + """ Process this deletion rule on the object. We use the CCDB passed by argument. Objects which have been created recently are spared (delay is expressed in minutes). This specific policy, 1_per_run, keeps only the most recent version for a given run based on the createdAt. @@ -20,13 +21,13 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t Config Parameters: - period_pass: Keep 1 version for a combination of run+pass+period if true. - delete_when_no_run: Versions without runs are preserved if delete_when_no_run is set to false (default). - Otherwise only the last one is preserved. + Otherwise, only the last one is preserved. THEY CANNOT BE BOTH TRUE AT THE SAME TIME It is implemented like this : Map of buckets: run[+pass+period] -> list of versions Go through all objects: Add the object to the corresponding key (run[+pass+period]) - If delete_when_no_run is false, remove the versions without run from the map + If delete_when_no_run is false, remove from the map the versions without run or with run==0 Go through the map: for each run (resp. run+pass+period) keep the most recent object and delete the rest. :param ccdb: the ccdb in which objects are cleaned up. @@ -36,8 +37,8 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t :param to_timestamp: only objects created before this timestamp are considered. :param extra_params: a dictionary containing extra parameters for this rule. :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved. - ''' - + """ + logger.debug(f"Plugin 1_per_run processing {object_path}") preservation_list: List[ObjectVersion] = [] @@ -46,11 +47,11 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t versions_buckets_dict: DefaultDict[str, List[ObjectVersion]] = defaultdict(list) # config parameters - delete_when_no_run = (extra_params.get("delete_when_no_run", False) == True) + delete_when_no_run = (extra_params.get("delete_when_no_run", False) is True) logger.debug(f"delete_when_no_run : {delete_when_no_run}") - period_pass = (extra_params.get("period_pass", False) == True) + period_pass = (extra_params.get("period_pass", False) is True) logger.debug(f"period_pass : {period_pass}") - if delete_when_no_run == True and period_pass == True: + if delete_when_no_run is True and period_pass is True: logger.error(f"1_per_run does not allow both delete_when_no_run and period_pass to be on at the same time") return {"deleted": 0, "preserved": 0, "updated": 0} @@ -64,6 +65,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t # if we should not touch the files with no runs, let's just remove them from the map if not delete_when_no_run: del versions_buckets_dict['none'] + del versions_buckets_dict['0'] # Dispatch the versions to deletion and preservation lists for bucket, run_versions in versions_buckets_dict.items(): @@ -92,10 +94,10 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t temp_deletion_list: List[ObjectVersion] = [] for v in deletion_list: if from_timestamp < v.validFrom < to_timestamp: # in the allowed period - temp_deletion_list.append(v) # we will delete any ways + temp_deletion_list.append(v) # we will delete any ways ccdb.deleteVersion(v) else: - preservation_list.append(v) # we preserve + preservation_list.append(v) # we preserve deletion_list = temp_deletion_list logger.debug(f"deleted ({len(deletion_list)}) : ") @@ -110,14 +112,12 @@ def process(ccdb: Ccdb, object_path: str, delay: int, from_timestamp: int, to_t for v in update_list: logger.debug(f" {v}") - return {"deleted" : len(deletion_list), "preserved": len(preservation_list), "updated" : len(update_list)} - - + return {"deleted": len(deletion_list), "preserved": len(preservation_list), "updated": len(update_list)} def main(): logger.getLogger().setLevel(int(10)) - dryable.set( True ) + dryable.set(True) ccdb = Ccdb('http://ccdb-test.cern.ch:8080') process(ccdb, "qc/testRunCleanup", 0)