From 6afac9d76dbc589554f789bbad54b3be99e0097f Mon Sep 17 00:00:00 2001 From: Benedikt Ziemons Date: Tue, 1 Dec 2020 11:05:34 +0100 Subject: [PATCH] Transfers: Add preparer daemon; Fix #4056 Searches for minimum distance source and sets QUEUED or WAITING state based on throttler mode and limits. Add PREPARING request state. Add conveyor.use_preparer config option. Add tests for preparer. --- bin/rucio-conveyor-preparer | 40 + etc/sql/oracle/schema.sql | 2 +- lib/rucio/alembicrevision.py | 2 +- lib/rucio/core/distance.py | 9 +- lib/rucio/core/request.py | 151 +- lib/rucio/core/transfer.py | 46 +- lib/rucio/daemons/conveyor/preparer.py | 145 ++ lib/rucio/db/sqla/constants.py | 2 + ...95260_extend_request_state_for_preparer.py | 65 + lib/rucio/tests/common.py | 2 + lib/rucio/tests/test_daemons.py | 9 +- lib/rucio/tests/test_preparer.py | 174 +++ lib/rucio/tests/test_request.py | 1304 ++--------------- lib/rucio/tests/test_throttler.py | 1236 ++++++++++++++-- tools/add_header | 5 +- 15 files changed, 1805 insertions(+), 1387 deletions(-) create mode 100755 bin/rucio-conveyor-preparer create mode 100644 lib/rucio/daemons/conveyor/preparer.py create mode 100644 lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py create mode 100644 lib/rucio/tests/test_preparer.py diff --git a/bin/rucio-conveyor-preparer b/bin/rucio-conveyor-preparer new file mode 100755 index 0000000000..a5377e5952 --- /dev/null +++ b/bin/rucio-conveyor-preparer @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -* +# Copyright 2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Benedikt Ziemons , 2020 + +import argparse +import signal + +from rucio.daemons.conveyor.preparer import run, stop + + +def main(args): + signal.signal(signal.SIGTERM, stop) + try: + run(once=args.run_once, threads=args.threads, sleep_time=args.sleep_time, bulk=args.bulk) + except KeyboardInterrupt: + stop() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--run-once", action="store_true", default=False, help='One iteration only') + parser.add_argument("--threads", action="store", default=1, type=int, help='Concurrency control: total number of threads on this process') + parser.add_argument("--sleep-time", action="store", default=60, type=int, help='Concurrency control: thread sleep time after each chunk of work') + parser.add_argument("--bulk", action="store", default=100, type=int, help='Limit of requests per chunk') + main(args=parser.parse_args()) diff --git a/etc/sql/oracle/schema.sql b/etc/sql/oracle/schema.sql index 3a40a3223a..3c7447165c 100644 --- a/etc/sql/oracle/schema.sql +++ b/etc/sql/oracle/schema.sql @@ -930,7 +930,7 @@ PARTITION BY LIST(SCOPE) CONSTRAINT REQUESTS_SCOPE_NN CHECK (SCOPE IS NOT NULL), CONSTRAINT REQUESTS_NAME_NN CHECK (NAME IS NOT NULL), CONSTRAINT REQUESTS_TYPE_CHK CHECK (request_type in ('U', 'D', 'T', 'I', 'O')), - CONSTRAINT REQUESTS_STATE_CHK CHECK (state IN ('Q', 'G', 'S', 'D', 'F', 'L', 'N', 'O', 'A', 'U', 'W', 'M')) + CONSTRAINT REQUESTS_STATE_CHK CHECK (state IN ('Q', 'G', 'S', 'D', 'F', 'L', 'N', 'O', 'A', 'U', 'W', 'M', 'P')) ) PCTFREE 3; diff --git a/lib/rucio/alembicrevision.py b/lib/rucio/alembicrevision.py index b949f50715..c1f4fe9dac 100644 --- a/lib/rucio/alembicrevision.py +++ b/lib/rucio/alembicrevision.py @@ -17,4 +17,4 @@ # - Benedikt Ziemons , 2020 # - Martin Barisits , 2020 -ALEMBIC_REVISION = '8ea9122275b1' # the current alembic head revision +ALEMBIC_REVISION = 'd23453595260' # the current alembic head revision diff --git a/lib/rucio/core/distance.py b/lib/rucio/core/distance.py index ab2c0ef9b5..5a86b8d1e6 100644 --- a/lib/rucio/core/distance.py +++ b/lib/rucio/core/distance.py @@ -20,8 +20,8 @@ # - Hannes Hansen , 2018-2019 # - Andrew Lister , 2019 # - Benedikt Ziemons , 2020 -# -# PY3K COMPATIBLE + +from typing import TYPE_CHECKING from sqlalchemy.exc import DatabaseError, IntegrityError from sqlalchemy.orm import aliased @@ -30,6 +30,9 @@ from rucio.db.sqla.models import Distance, RSE from rucio.db.sqla.session import transactional_session, read_session +if TYPE_CHECKING: + from typing import List, Dict + @transactional_session def add_distance(src_rse_id, dest_rse_id, ranking=None, agis_distance=None, geoip_distance=None, @@ -77,7 +80,7 @@ def add_distance_short(src_rse_id, dest_rse_id, distance=None, session=None): @read_session -def get_distances(src_rse_id=None, dest_rse_id=None, session=None): +def get_distances(src_rse_id=None, dest_rse_id=None, session=None) -> "List[Dict]": """ Get distances between rses. diff --git a/lib/rucio/core/request.py b/lib/rucio/core/request.py index 8a6b047097..7c32736a0c 100644 --- a/lib/rucio/core/request.py +++ b/lib/rucio/core/request.py @@ -27,32 +27,37 @@ # - Hannes Hansen , 2018-2019 # - Andrew Lister , 2019 # - Brandon White , 2019 +# - Benedikt Ziemons , 2020 import datetime import json import logging import time import traceback +from typing import TYPE_CHECKING from six import string_types - from sqlalchemy import and_, or_, func, update from sqlalchemy.exc import IntegrityError -# from sqlalchemy.sql import tuple_ from sqlalchemy.sql.expression import asc, false, true +from rucio.common.config import config_get_bool from rucio.common.exception import RequestNotFound, RucioException, UnsupportedOperation, ConfigNotFound from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid, chunks, get_parsed_throttler_mode from rucio.core.config import get from rucio.core.message import add_message from rucio.core.monitor import record_counter, record_timer -from rucio.core.rse import get_rse_name, get_rse_transfer_limits, get_rse_vo +from rucio.core.rse import get_rse_name, get_rse_vo, get_rse_transfer_limits from rucio.db.sqla import models, filter_thread_work from rucio.db.sqla.constants import RequestState, RequestType, FTSState, ReplicaState, LockState, RequestErrMsg from rucio.db.sqla.session import read_session, transactional_session, stream_session from rucio.transfertool.fts3 import FTS3Transfertool +if TYPE_CHECKING: + from typing import List, Tuple, Iterable, Iterator, Optional + from sqlalchemy.orm import Session + """ The core request.py is specifically for handling requests. Requests accessed by external_id (So called transfers), are covered in the core transfer.py @@ -135,7 +140,8 @@ def queue_requests(requests, session=None): logging.debug("queue requests") request_clause = [] - transfer_limits, rses = get_rse_transfer_limits(session=session), {} + rses = {} + preparer_enabled = config_get_bool('conveyor', 'use_preparer', raise_exception=False, default=False) for req in requests: if isinstance(req['attributes'], string_types): @@ -166,35 +172,6 @@ def queue_requests(requests, session=None): for request in query_existing_requests: existing_requests.append(request) - # Temporary disabled - source_rses = {} - # request_scopes_names = [(request['scope'], request['name']) for request in requests] - # for chunked_requests in chunks(request_scopes_names, 50): - # results = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id, models.Distance.dest_rse_id, models.Distance.ranking)\ - # .filter(tuple_(models.RSEFileAssociation.scope, models.RSEFileAssociation.name).in_(chunked_requests))\ - # .join(models.Distance, models.Distance.src_rse_id == models.RSEFileAssociation.rse_id)\ - # .all() - # for result in results: - # scope = result[0] - # name = result[1] - # src_rse_id = result[2] - # dest_rse_id = result[3] - # distance = result[4] - # if scope not in source_rses: - # source_rses[scope] = {} - # if name not in source_rses[scope]: - # source_rses[scope][name] = {} - # if dest_rse_id not in source_rses[scope][name]: - # source_rses[scope][name][dest_rse_id] = {} - # if src_rse_id not in source_rses[scope][name][dest_rse_id]: - # source_rses[scope][name][dest_rse_id][src_rse_id] = distance - - try: - throttler_mode = get('throttler', 'mode', default=None, use_cache=False, session=session) - direction, all_activities = get_parsed_throttler_mode(throttler_mode) - except ConfigNotFound: - throttler_mode = None - new_requests, sources, messages = [], [], [] for request in requests: dest_rse_name = get_rse_name(rse_id=request['dest_rse_id'], session=session) @@ -210,38 +187,13 @@ def temp_serializer(obj): return obj.internal raise TypeError('Could not serialise object %r' % obj) - source_rse_id = request.get('source_rse_id') - if not source_rse_id: - try: - source_rses_of_request = source_rses[request['scope']][request['name']][request['dest_rse_id']] - source_rse_id = min(source_rses_of_request, key=source_rses_of_request.get) - except KeyError: - pass - activity_limit = transfer_limits.get(request['attributes']['activity'], {}) - all_activities_limit = transfer_limits.get('all_activities', {}) - limit_found = False - if throttler_mode: - if direction == 'source': - if all_activities: - if all_activities_limit.get(source_rse_id): - limit_found = True - else: - if activity_limit.get(source_rse_id): - limit_found = True - elif direction == 'destination': - if all_activities: - if all_activities_limit.get(request['dest_rse_id']): - limit_found = True - else: - if activity_limit.get(request['dest_rse_id']): - limit_found = True - request['state'] = RequestState.WAITING if limit_found else RequestState.QUEUED + request['state'] = RequestState.PREPARING if preparer_enabled else RequestState.QUEUED new_request = {'request_type': request['request_type'], 'scope': request['scope'], 'name': request['name'], 'dest_rse_id': request['dest_rse_id'], - 'source_rse_id': source_rse_id, + 'source_rse_id': request.get('source_rse_id', None), 'attributes': json.dumps(request['attributes'], default=temp_serializer), 'state': request['state'], 'rule_id': request['rule_id'], @@ -304,6 +256,7 @@ def temp_serializer(obj): for messages_chunk in chunks(messages, 1000): session.bulk_insert_mappings(models.Message, messages_chunk) + return new_requests @@ -1539,3 +1492,81 @@ def list_requests(src_rse_ids, dst_rse_ids, states=[RequestState.WAITING], sessi models.Request.dest_rse_id.in_(dst_rse_ids)) for request in query.yield_per(500): yield request + + +# important column indices from __list_transfer_requests_and_source_replicas +request_id_col = 0 +activity_col = 7 +dest_rse_id_col = 10 +source_rse_id_col = 12 +distance_col = 20 + + +@transactional_session +def preparer_update_requests(source_iter: "Iterable[Tuple]", session: "Optional[Session]" = None) -> int: + count = 0 + for req_source in source_iter: + new_state = __throttler_request_state( + activity=req_source[activity_col], + source_rse_id=req_source[source_rse_id_col], + dest_rse_id=req_source[dest_rse_id_col], + session=session, + ) + session.query(models.Request).filter_by(id=req_source[request_id_col]).update({ + models.Request.state: new_state, + models.Request.source_rse_id: req_source[source_rse_id_col], + }, synchronize_session=False) + count += 1 + return count + + +def __throttler_request_state(activity, source_rse_id, dest_rse_id, session: "Optional[Session]" = None) -> RequestState: + """ + Takes request attributes to return a new state for the request + based on throttler settings. Always returns QUEUED, + if the throttler mode is not set. + """ + try: + throttler_mode = get('throttler', 'mode', default=None, use_cache=False, session=session) + except ConfigNotFound: + throttler_mode = None + + limit_found = False + if throttler_mode: + transfer_limits = get_rse_transfer_limits(session=session) + activity_limit = transfer_limits.get(activity, {}) + all_activities_limit = transfer_limits.get('all_activities', {}) + direction, all_activities = get_parsed_throttler_mode(throttler_mode) + if direction == 'source': + if all_activities: + if all_activities_limit.get(source_rse_id): + limit_found = True + else: + if activity_limit.get(source_rse_id): + limit_found = True + elif direction == 'destination': + if all_activities: + if all_activities_limit.get(dest_rse_id): + limit_found = True + else: + if activity_limit.get(dest_rse_id): + limit_found = True + + return RequestState.WAITING if limit_found else RequestState.QUEUED + + +def minimum_distance_requests(req_sources: "List[Tuple]") -> "Iterator[Tuple]": + # sort by Request.id, should be pretty quick since the database should return it this way (tim sort) + req_sources.sort(key=lambda t: t[request_id_col]) + + # req_sources must be non-empty and sorted by ids at this point, see above + cur_request_id = req_sources[0][request_id_col] + shortest_item = req_sources[0] + for idx in range(1, len(req_sources)): + if cur_request_id != req_sources[idx][request_id_col]: + yield shortest_item + cur_request_id = req_sources[idx][request_id_col] + shortest_item = req_sources[idx] + elif req_sources[idx][distance_col] < shortest_item[distance_col]: + shortest_item = req_sources[idx] + yield shortest_item diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py index e59c84bec0..7915e91f1b 100644 --- a/lib/rucio/core/transfer.py +++ b/lib/rucio/core/transfer.py @@ -32,9 +32,7 @@ # - Eli Chadwick , 2020 # - Nick Smith , 2020 # - Patrick Austin , 2020 -# -# PY3K COMPATIBLE - +# - Benedikt Ziemons , 2020 from __future__ import division @@ -46,6 +44,7 @@ import re import time import traceback +from typing import TYPE_CHECKING from dogpile.cache import make_region from dogpile.cache.api import NoValue @@ -54,14 +53,14 @@ from sqlalchemy.sql.expression import false from rucio.common import constants +from rucio.common.config import config_get +from rucio.common.constants import SUPPORTED_PROTOCOLS from rucio.common.exception import (InvalidRSEExpression, NoDistance, RequestNotFound, RSEProtocolNotSupported, RucioException, UnsupportedOperation) -from rucio.common.config import config_get from rucio.common.rse_attributes import get_rse_attributes from rucio.common.types import InternalAccount from rucio.common.utils import construct_surl -from rucio.common.constants import SUPPORTED_PROTOCOLS from rucio.core import did, message as message_core, request as request_core from rucio.core.config import get as core_config_get from rucio.core.monitor import record_counter, record_timer @@ -76,6 +75,8 @@ from rucio.rse import rsemanager as rsemgr from rucio.transfertool.fts3 import FTS3Transfertool +if TYPE_CHECKING: + from typing import List, Tuple # Extra modules: Only imported if available EXTRA_MODULES = {'globus_sdk': False} @@ -650,6 +651,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, activity=activity, older_than=older_than, rses=rses, + request_state=RequestState.QUEUED, session=session) unavailable_read_rse_ids = __get_unavailable_rse_ids(operation='read', session=session) @@ -1267,8 +1269,8 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, @read_session -def __list_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, - limit=None, activity=None, older_than=None, rses=None, session=None): +def __list_transfer_requests_and_source_replicas(total_workers=0, worker_number=0, limit=None, activity=None, + older_than=None, rses=None, request_state=None, session=None) -> "List[Tuple]": """ List requests with source replicas :param total_workers: Number of total workers. @@ -1280,6 +1282,10 @@ def __list_transfer_requests_and_source_replicas(total_workers=0, worker_number= :param session: Database session to use. :returns: List. """ + + if request_state is None: + request_state = RequestState.QUEUED + sub_requests = session.query(models.Request.id, models.Request.rule_id, models.Request.scope, @@ -1292,12 +1298,12 @@ def __list_transfer_requests_and_source_replicas(total_workers=0, worker_number= models.Request.previous_attempt_id, models.Request.dest_rse_id, models.Request.retry_count, - models.Request.account)\ - .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_TYP_STA_UPD_IDX)", 'oracle')\ - .filter(models.Request.state == RequestState.QUEUED)\ - .filter(models.Request.request_type == RequestType.TRANSFER)\ - .join(models.RSE, models.RSE.id == models.Request.dest_rse_id)\ - .filter(models.RSE.deleted == false())\ + models.Request.account) \ + .with_hint(models.Request, "INDEX(REQUESTS REQUESTS_TYP_STA_UPD_IDX)", 'oracle') \ + .filter(models.Request.state == request_state) \ + .filter(models.Request.request_type == RequestType.TRANSFER) \ + .join(models.RSE, models.RSE.id == models.Request.dest_rse_id) \ + .filter(models.RSE.deleted == false()) \ .filter(models.RSE.availability.in_((2, 3, 6, 7))) if isinstance(older_than, datetime.datetime): @@ -1333,19 +1339,19 @@ def __list_transfer_requests_and_source_replicas(total_workers=0, worker_number= sub_requests.c.retry_count, models.Source.url, models.Source.ranking, - models.Distance.ranking)\ + models.Distance.ranking) \ .outerjoin(models.RSEFileAssociation, and_(sub_requests.c.scope == models.RSEFileAssociation.scope, sub_requests.c.name == models.RSEFileAssociation.name, models.RSEFileAssociation.state == ReplicaState.AVAILABLE, - sub_requests.c.dest_rse_id != models.RSEFileAssociation.rse_id))\ - .with_hint(models.RSEFileAssociation, "+ index(replicas REPLICAS_PK)", 'oracle')\ + sub_requests.c.dest_rse_id != models.RSEFileAssociation.rse_id)) \ + .with_hint(models.RSEFileAssociation, "+ index(replicas REPLICAS_PK)", 'oracle') \ .outerjoin(models.RSE, and_(models.RSE.id == models.RSEFileAssociation.rse_id, - models.RSE.deleted == false()))\ + models.RSE.deleted == false())) \ .outerjoin(models.Source, and_(sub_requests.c.id == models.Source.request_id, - models.RSE.id == models.Source.rse_id))\ - .with_hint(models.Source, "+ index(sources SOURCES_PK)", 'oracle')\ + models.RSE.id == models.Source.rse_id)) \ + .with_hint(models.Source, "+ index(sources SOURCES_PK)", 'oracle') \ .outerjoin(models.Distance, and_(sub_requests.c.dest_rse_id == models.Distance.dest_rse_id, - models.RSEFileAssociation.rse_id == models.Distance.src_rse_id))\ + models.RSEFileAssociation.rse_id == models.Distance.src_rse_id)) \ .with_hint(models.Distance, "+ index(distances DISTANCES_PK)", 'oracle') if rses: diff --git a/lib/rucio/daemons/conveyor/preparer.py b/lib/rucio/daemons/conveyor/preparer.py new file mode 100644 index 0000000000..af71b623d6 --- /dev/null +++ b/lib/rucio/daemons/conveyor/preparer.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Benedikt Ziemons , 2020 + +import logging +import os +import socket +import sys +import threading +from time import time +from typing import TYPE_CHECKING + +import rucio.db.sqla.util +from rucio.common import exception +from rucio.common.config import config_get +from rucio.common.exception import RucioException +from rucio.core import heartbeat +from rucio.core.request import preparer_update_requests, minimum_distance_requests +from rucio.core.transfer import __list_transfer_requests_and_source_replicas +from rucio.db.sqla.constants import RequestState + +if TYPE_CHECKING: + from typing import Optional + from sqlalchemy.orm import Session + +graceful_stop = threading.Event() + + +def stop(): + """ + Graceful exit. + """ + + graceful_stop.set() + + +def run(once=False, threads=1, sleep_time=10, bulk=100): + """ + Running the preparer daemon either once or by default in a loop until stop is called. + """ + config_loglevel = config_get('common', 'loglevel', raise_exception=False, default='DEBUG').upper() + logging.basicConfig(stream=sys.stdout, + level=config_loglevel, + format='%(asctime)s\t%(process)d\t%(levelname)s\t%(message)s') + + if rucio.db.sqla.util.is_old_db(): + raise exception.DatabaseException('Database was not updated, daemon won\'t start') + + def preparer_kwargs(): + # not sure if this is needed for threading.Thread, but it always returns a fresh dictionary + return {'once': once, 'sleep_time': sleep_time, 'bulk': bulk} + + threads = [threading.Thread(target=preparer, name=f'conveyor-preparer-{i}', kwargs=preparer_kwargs(), daemon=True) for i in range(threads)] + for thr in threads: + thr.start() + + all_running = True + while all_running: + for thr in threads: + thr.join(timeout=3.14) + if not thr.is_alive() or graceful_stop.is_set(): + all_running = False + break + + if graceful_stop.is_set() or once: + logging.info('conveyor-preparer: gracefully stopping') + else: + logging.warning('conveyor-preparer: stopping out of the ordinary') + graceful_stop.set() + + for thr in threads: + thr.join(timeout=3.14) + + logging.info('conveyor-preparer: stopped') + + +def preparer(once, sleep_time, bulk): + # Make an initial heartbeat so that all instanced daemons have the correct worker number on the next try + executable = 'conveyor-preparer' + hostname = socket.gethostname() + pid = os.getpid() + current_thread = threading.current_thread() + worker_number = current_thread + total_workers = '?' + heartbeat.sanity_check(executable=executable, hostname=hostname, pid=pid, thread=current_thread) + + try: + graceful_stop.wait(10) # gathering of daemons/threads on first start + while not graceful_stop.is_set(): + start_time = time() + + pulse = heartbeat.live(executable=executable, hostname=hostname, pid=pid, thread=current_thread) + worker_number = pulse['assign_thread'] + total_workers = pulse['nr_threads'] + + try: + updated_msg = run_once(total_workers=total_workers, worker_number=worker_number, limit=bulk) + except RucioException: + logging.exception('conveyor-preparer[%s/%s] errored with a RucioException, retrying later' % (worker_number, total_workers)) + updated_msg = 'errored' + + if once: + break + + end_time = time() + time_diff = end_time - start_time + logging.info('conveyor-preparer[%s/%s] %s, taking %.3f seconds' % (worker_number, total_workers, updated_msg, time_diff)) + if time_diff < sleep_time: + sleep_remaining = sleep_time - time_diff + logging.info('conveyor-preparer[%s/%s] sleeping for a while : %.2f seconds' % (worker_number, total_workers, sleep_remaining)) + graceful_stop.wait(sleep_remaining) + + logging.info('conveyor-preparer[%s/%s]: gracefully stopping' % (worker_number, total_workers)) + + finally: + heartbeat.die(executable=executable, hostname=hostname, pid=pid, thread=current_thread) + + +def run_once(total_workers: int = 0, worker_number: int = 0, limit: "Optional[int]" = None, session: "Optional[Session]" = None) -> str: + req_sources = __list_transfer_requests_and_source_replicas( + total_workers=total_workers, + worker_number=worker_number, + limit=limit, + request_state=RequestState.PREPARING, + session=session + ) + if not req_sources: + return 'had nothing to do' + + count = preparer_update_requests(minimum_distance_requests(req_sources), session=session) + return f'updated {count}/{limit} requests' diff --git a/lib/rucio/db/sqla/constants.py b/lib/rucio/db/sqla/constants.py index 3e860c6c25..a370a45dd6 100644 --- a/lib/rucio/db/sqla/constants.py +++ b/lib/rucio/db/sqla/constants.py @@ -22,6 +22,7 @@ # - Ruturaj Gujar , 2019 # - Jaroslav Guenther , 2019 # - Mario Lassnig , 2020 +# - Benedikt Ziemons , 2020 from datetime import datetime from enum import Enum @@ -159,6 +160,7 @@ class RequestState(Enum): MISMATCH_SCHEME = 'M' SUSPEND = 'U' WAITING = 'W' + PREPARING = 'P' class RequestType(Enum): diff --git a/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py b/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py new file mode 100644 index 0000000000..8e0ba0b41b --- /dev/null +++ b/lib/rucio/db/sqla/migrate_repo/versions/d23453595260_extend_request_state_for_preparer.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Benedikt Ziemons , 2020 + +""" +Add PREPARING state to Request model. +""" + +from alembic import context +from alembic.op import create_check_constraint, drop_constraint + +# Alembic revision identifiers +revision = 'd23453595260' +down_revision = '8ea9122275b1' + + +def upgrade(): + """ + Upgrade the database to this revision + """ + + new_enum_values = ['Q', 'G', 'S', 'D', 'F', 'L', 'N', 'O', 'A', 'U', 'W', 'M', 'P'] + update_enum(new_enum_values) + + +def downgrade(): + """ + Downgrade the database to the previous revision + """ + + old_enum_values = ['Q', 'G', 'S', 'D', 'F', 'L', 'N', 'O', 'A', 'U', 'W', 'M'] + update_enum(old_enum_values) + + +def enum_values_str(enumvals): + return ', '.join(map(lambda x: x.join(("'", "'")), enumvals)) + + +def update_enum(enum_values): + dialect = context.get_context().dialect.name + create_check = False + + if dialect in ['oracle', 'postgresql']: + drop_constraint('REQUESTS_STATE_CHK', 'requests', type_='check') + create_check = True + elif dialect == 'mysql': + create_check = True + + if create_check: + create_check_constraint(constraint_name='REQUESTS_STATE_CHK', table_name='requests', + condition=f'state in ({enum_values_str(enum_values)})') diff --git a/lib/rucio/tests/common.py b/lib/rucio/tests/common.py index adc4e55b0c..ce33ce7093 100644 --- a/lib/rucio/tests/common.py +++ b/lib/rucio/tests/common.py @@ -40,6 +40,8 @@ from rucio.common.utils import generate_uuid as uuid, execute skip_rse_tests_with_accounts = pytest.mark.skipif(not os.path.exists('etc/rse-accounts.cfg'), reason='fails if no rse-accounts.cfg found') +skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and (os.environ['RDBMS'] == 'sqlite' or os.environ['RDBMS'] == 'mysql5'), + reason="does not work in SQLite or MySQL 5, because of missing features") def account_name_generator(): diff --git a/lib/rucio/tests/test_daemons.py b/lib/rucio/tests/test_daemons.py index 34eae15425..bc29114457 100644 --- a/lib/rucio/tests/test_daemons.py +++ b/lib/rucio/tests/test_daemons.py @@ -15,8 +15,6 @@ # # Authors: # - Benedikt Ziemons , 2020 -# -# PY3K COMPATIBLE import sys @@ -30,7 +28,7 @@ from rucio.daemons.badreplicas import minos, minos_temporary_expiration, necromancer from rucio.daemons.c3po import c3po from rucio.daemons.cache import consumer -from rucio.daemons.conveyor import finisher, fts_throttler, poller, poller_latest, receiver, stager, submitter, throttler +from rucio.daemons.conveyor import finisher, fts_throttler, poller, poller_latest, receiver, stager, submitter, throttler, preparer from rucio.daemons.follower import follower from rucio.daemons.hermes import hermes, hermes2 from rucio.daemons.judge import cleaner, evaluator, injector, repairer @@ -67,6 +65,7 @@ stager, submitter, throttler, + preparer, follower, hermes, hermes2, @@ -87,8 +86,10 @@ undertaker, ] +ids = [mod.__name__ for mod in DAEMONS] + -@pytest.mark.parametrize('daemon', DAEMONS) +@pytest.mark.parametrize('daemon', argvalues=DAEMONS, ids=ids) @mock.patch('rucio.db.sqla.util.is_old_db') def test_fail_on_old_database(mock_is_old_db, daemon): """ DAEMON: Test daemon failure on old database """ diff --git a/lib/rucio/tests/test_preparer.py b/lib/rucio/tests/test_preparer.py new file mode 100644 index 0000000000..ed3547395d --- /dev/null +++ b/lib/rucio/tests/test_preparer.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 CERN +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# - Benedikt Ziemons , 2020 + +import pytest + +from rucio.common.types import InternalScope, InternalAccount +from rucio.common.utils import generate_uuid +from rucio.core import config +from rucio.core.distance import get_distances, add_distance +from rucio.core.replica import add_replicas, delete_replicas +from rucio.core.rse import get_rse_id, set_rse_transfer_limits, add_rse, del_rse +from rucio.core.transfer import __list_transfer_requests_and_source_replicas +from rucio.daemons.conveyor import preparer +from rucio.db.sqla import models, session +from rucio.db.sqla.constants import RequestState +from rucio.tests.common import rse_name_generator + + +@pytest.fixture +def db_session(): + db_session = session.get_session() + yield db_session + db_session.rollback() + + +@pytest.fixture(scope='module') +def dest_rse(vo): + dest_rse = 'MOCK' + dest_rse_id = get_rse_id(dest_rse, vo=vo) + return {'name': dest_rse, 'id': dest_rse_id} + + +def generate_rse(vo='def', session=None): + rse_name = f'MOCK-{rse_name_generator()}' + rse_id = add_rse(rse_name, vo=vo, session=session) + return {'name': rse_name, 'id': rse_id} + + +@pytest.fixture +def source_rse(db_session, vo, dest_rse): + rse = generate_rse(vo=vo, session=db_session) + add_distance(rse['id'], dest_rse['id'], ranking=5, session=db_session) + db_session.commit() + + yield rse + + del_rse(rse['id'], session=db_session) + db_session.commit() + + +@pytest.fixture +def file(vo): + scope = InternalScope(scope='mock', vo=vo) + name = generate_uuid() + return {'scope': scope, 'name': name, 'bytes': 1, 'adler32': 'deadbeef'} + + +@pytest.fixture +def mock_request(db_session, vo, source_rse, dest_rse, file): + account = InternalAccount('root', vo=vo) + + add_replicas(rse_id=source_rse['id'], files=[file], account=account, session=db_session) + + request = models.Request(state=RequestState.PREPARING, scope=file['scope'], name=file['name'], dest_rse_id=dest_rse['id'], account=account) + request.save(session=db_session) + db_session.commit() + + yield request + + request.delete(session=db_session) + delete_replicas(rse_id=source_rse['id'], files=[file], session=db_session) + db_session.commit() + + +@pytest.fixture +def dest_throttler(db_session, mock_request): + config.set('throttler', 'mode', 'DEST_PER_ACT', session=db_session) + set_rse_transfer_limits(mock_request.dest_rse_id, activity=mock_request.activity, max_transfers=1, strategy='fifo', session=db_session) + db_session.commit() + + yield + + db_session.query(models.RSETransferLimit).filter_by(rse_id=mock_request.dest_rse_id).delete() + config.remove_option('throttler', 'mode', session=db_session) + db_session.commit() + + +def test_listing_preparing_transfers(db_session, mock_request): + req_sources = __list_transfer_requests_and_source_replicas(request_state=RequestState.PREPARING, session=db_session) + + assert len(req_sources) == 1 + req_id = req_sources[0][0] + assert req_id == mock_request.id + + +@pytest.mark.usefixtures('dest_throttler') +def test_preparer_setting_request_state_waiting(db_session, mock_request): + preparer.run_once(session=db_session) + db_session.commit() + + updated_mock_request = db_session.query(models.Request).filter_by(id=mock_request.id).one() # type: models.Request + + assert updated_mock_request.state == RequestState.WAITING + + +def test_preparer_setting_request_state_queued(db_session, mock_request): + preparer.run_once(session=db_session) + db_session.commit() + + updated_mock_request = db_session.query(models.Request).filter_by(id=mock_request.id).one() # type: models.Request + + assert updated_mock_request.state == RequestState.QUEUED + + +def test_preparer_setting_request_source(db_session, vo, source_rse, mock_request): + preparer.run_once(session=db_session) + db_session.commit() + + updated_mock_request = db_session.query(models.Request).filter_by(id=mock_request.id).one() # type: models.Request + + assert updated_mock_request.state == RequestState.QUEUED + assert updated_mock_request.source_rse_id == source_rse['id'] + + +@pytest.fixture +def source2_rse(db_session, vo, dest_rse): + rse = generate_rse(vo=vo) + add_distance(rse['id'], dest_rse['id'], ranking=2, session=db_session) + db_session.commit() + + yield rse + + del_rse(rse['id'], session=db_session) + db_session.commit() + + +def test_two_sources_one_destination(db_session, vo, file, source_rse, source2_rse, mock_request): + add_replicas(rse_id=source2_rse['id'], files=[file], account=mock_request.account, session=db_session) + try: + src1_distance, src2_distance = (get_distances( + src_rse_id=src_rse, + dest_rse_id=mock_request.dest_rse_id, + session=db_session + ) for src_rse in (source_rse['id'], source2_rse['id'])) + + assert src1_distance and len(src1_distance) == 1 and src1_distance[0]['ranking'] == 5 + assert src2_distance and len(src2_distance) == 1 and src2_distance[0]['ranking'] == 2 + + preparer.run_once(session=db_session) + db_session.commit() + + updated_mock_request = db_session.query(models.Request).filter_by(id=mock_request.id).one() # type: models.Request + + assert updated_mock_request.state == RequestState.QUEUED + assert updated_mock_request.source_rse_id == source2_rse['id'] # distance 2 < 5 + + finally: + delete_replicas(rse_id=source2_rse['id'], files=[file], session=db_session) + db_session.commit() diff --git a/lib/rucio/tests/test_request.py b/lib/rucio/tests/test_request.py index 5efb978d97..c8e094b6b3 100644 --- a/lib/rucio/tests/test_request.py +++ b/lib/rucio/tests/test_request.py @@ -20,1223 +20,127 @@ # - Eli Chadwick , 2020 # - Patrick Austin , 2020 # - Benedikt Ziemons , 2020 -# -# PY3K COMPATIBLE -import os import unittest from datetime import datetime import pytest -from rucio.common.config import config_get, config_get_bool +from rucio.common.config import config_get, config_get_bool, config_set, config_remove_option from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid, parse_response -from rucio.core.config import set as config_set -from rucio.core.did import attach_dids, add_did from rucio.core.replica import add_replica -from rucio.core.request import release_all_waiting_requests, queue_requests, get_request_by_did, release_waiting_requests_per_free_volume, \ - release_waiting_requests_grouped_fifo, release_waiting_requests_fifo, list_requests, release_waiting_requests_per_deadline +from rucio.core.request import queue_requests, get_request_by_did, list_requests from rucio.core.rse import get_rse_id, set_rse_transfer_limits, add_rse_attribute from rucio.db.sqla import session, models, constants +from rucio.db.sqla.constants import RequestType, RequestState from rucio.tests.common import vohdr, hdrdict, headers, auth -skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and (os.environ['RDBMS'] == 'sqlite' or os.environ['RDBMS'] == 'mysql5'), reason="does not work in SQLite or MySQL 5, because of missing features") - - -class TestRequestCoreQueue(unittest.TestCase): - - @classmethod - def setUpClass(cls): - if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): - cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} - else: - cls.vo = {} - - cls.db_session = session.get_session() - cls.dialect = cls.db_session.bind.dialect.name - cls.dest_rse = 'MOCK' - cls.dest_rse2 = 'MOCK2' - cls.source_rse = 'MOCK4' - cls.source_rse2 = 'MOCK5' - cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo) - cls.dest_rse_id2 = get_rse_id(cls.dest_rse2, **cls.vo) - cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo) - cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo) - cls.scope = InternalScope('mock', **cls.vo) - cls.account = InternalAccount('root', **cls.vo) - cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo) - cls.user_activity = 'User Subscription' - - def setUp(self): - self.db_session.query(models.Source).delete() - self.db_session.query(models.Request).delete() - self.db_session.query(models.RSETransferLimit).delete() - self.db_session.query(models.Distance).delete() - self.db_session.query(models.Config).delete() - self.db_session.commit() - - def tearDown(self): - self.db_session.commit() - - def test_queue_requests_state_no_throttler(self): - """ REQUEST (CORE): queue requests with default throttler mode (None). """ - name = generate_uuid() - name2 = generate_uuid() - name3 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id2, self.scope, name2, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - - set_rse_transfer_limits(self.dest_rse_id, self.user_activity, max_transfers=1, session=self.db_session) - set_rse_transfer_limits(self.dest_rse_id2, self.user_activity, max_transfers=1, session=self.db_session) - set_rse_transfer_limits(self.source_rse_id, self.user_activity, max_transfers=1, session=self.db_session) - set_rse_transfer_limits(self.source_rse_id2, self.user_activity, max_transfers=1, session=self.db_session) - - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'src_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 10, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'src_rse_id': self.source_rse_id2, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': 'unknown', - 'bytes': 10, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id2, - 'src_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name3, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 10, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name3, self.dest_rse_id2, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - # def test_queue_requests_source_rse(self): - # """ REQUEST (CORE): queue requests and select correct source RSE. """ - # # test correct selection of source RSE - # name = generate_uuid() - # size = 8 - # add_replica(self.source_rse_id, self.scope, name, size, self.account, session=self.db_session) - # add_replica(self.source_rse_id2, self.scope, name, size, self.account, session=self.db_session) - # add_distance(self.source_rse_id, self.dest_rse_id, 1, session=self.db_session) - # add_distance(self.source_rse_id2, self.dest_rse_id, 2, session=self.db_session) - # requests = [{ - # 'dest_rse_id': self.dest_rse_id, - # 'request_type': constants.RequestType.TRANSFER, - # 'request_id': generate_uuid(), - # 'name': name, - # 'scope': self.scope, - # 'rule_id': generate_uuid(), - # 'retry_count': 1, - # 'requested_at': datetime.now().replace(year=2015), - # 'attributes': { - # 'activity': self.user_activity, - # 'bytes': size, - # 'md5': '', - # 'adler32': '' - # } - # }] - # queue_requests(requests, session=self.db_session) - # request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) - # # select source RSE with smallest distance - # assert request['source_rse_id'] == self.source_rse_id - # assert request['name'] == name - # assert request['scope'] == self.scope - # assert request['state'] == constants.RequestState.QUEUED - - def test_queue_requests_state_1(self): - """ REQUEST (CORE): queue requests and set correct request state. """ - # test correct request state depending on throttler mode - config_set('throttler', 'mode', 'DEST_PER_ACT', session=self.db_session) - set_rse_transfer_limits(self.dest_rse_id, self.user_activity, max_transfers=1, session=self.db_session) - size = 1 - name = generate_uuid() - add_replica(self.source_rse_id2, self.scope, name, size, self.account, session=self.db_session) - name2 = generate_uuid() - add_replica(self.source_rse_id2, self.scope, name2, size, self.account, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': self.user_activity, - 'bytes': size, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': 'Activity without limit', - 'bytes': size, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - def test_queue_requests_state_2(self): - """ REQUEST (CORE): queue requests and set correct request state. """ - config_set('throttler', 'mode', 'SRC_PER_ACT', session=self.db_session) - size = 1 - name = generate_uuid() - add_replica(self.source_rse_id2, self.scope, name, size, self.account, session=self.db_session) - name2 = generate_uuid() - add_replica(self.source_rse_id2, self.scope, name2, size, self.account, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': self.user_activity, - 'bytes': size, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - -class TestRequestCoreRelease(unittest.TestCase): - - @classmethod - def setUpClass(cls): - if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): - cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} - else: - cls.vo = {} - - cls.db_session = session.get_session() - cls.dialect = cls.db_session.bind.dialect.name - cls.dest_rse = 'MOCK' - cls.source_rse = 'MOCK4' - cls.source_rse2 = 'MOCK5' - cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo) - cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo) - cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo) - cls.scope = InternalScope('mock', **cls.vo) - cls.account = InternalAccount('root', **cls.vo) - cls.user_activity = 'User Subscription' - cls.all_activities = 'all_activities' - - def setUp(self): - self.db_session.query(models.Request).delete() - self.db_session.query(models.RSETransferLimit).delete() - self.db_session.query(models.Distance).delete() - self.db_session.query(models.Config).delete() - # set transfer limits to put requests in waiting state - set_rse_transfer_limits(self.dest_rse_id, self.user_activity, max_transfers=1, session=self.db_session) - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, max_transfers=1, session=self.db_session) - set_rse_transfer_limits(self.dest_rse_id, 'ignore', max_transfers=1, session=self.db_session) - config_set('throttler', 'mode', 'DEST_PER_ACT', session=self.db_session) - self.db_session.commit() - - def tearDown(self): - self.db_session.commit() - - @skiplimitedsql - def test_release_waiting_requests_per_free_volume(self): - """ REQUEST (CORE): release waiting requests that fit grouped in available volume.""" - # release unattached requests that fit in available volume with respect to already submitted transfers - name1 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - name3 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.all_activities, state=constants.RequestState.SUBMITTED) - request.save(session=self.db_session) - volume = 10 - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=volume, max_transfers=1, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': 'User Subscription', - 'bytes': 8, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'requested_at': datetime.now().replace(year=2020), - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': 'User Subscription', - 'bytes': 2, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name3, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2000), - 'attributes': { - 'activity': 'User Subscription', - 'bytes': 10, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_per_free_volume(self.dest_rse_id, volume=volume, session=self.db_session) - # released because small enough - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - # still waiting because requested later and to big - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - # still waiting because too big - request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - - # release attached requests that fit together with the dataset in available volume with respect to already submitted transfers - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - name3 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - name4 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) - dataset1_name = generate_uuid() - add_did(self.scope, dataset1_name, constants.DIDType.DATASET, self.account, session=self.db_session) - attach_dids(self.scope, dataset1_name, [{'name': name1, 'scope': self.scope}, {'name': name4, 'scope': self.scope}], self.account, session=self.db_session) - dataset2_name = generate_uuid() - add_did(self.scope, dataset2_name, constants.DIDType.DATASET, self.account, session=self.db_session) - attach_dids(self.scope, dataset2_name, [{'name': name2, 'scope': self.scope}, {'name': name3, 'scope': self.scope}], self.account, session=self.db_session) - request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.all_activities, state=constants.RequestState.SUBMITTED) - request.save(session=self.db_session) - volume = 10 - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=volume, max_transfers=1, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 6, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'requested_at': datetime.now().replace(year=2020), - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 2, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name3, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2000), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 10, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name4, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2030), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 2, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_per_free_volume(self.dest_rse_id, volume=volume, session=self.db_session) - # released because dataset fits in volume - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - # waiting because dataset is too big - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - - # release requests with no available volume -> release nothing - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - add_replica(self.dest_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - volume = 0 - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=volume, max_transfers=1, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2015), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 8, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_per_free_volume(self.dest_rse_id, volume=volume, session=self.db_session) - # waiting because no available volume - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - - @skiplimitedsql - def test_release_waiting_requests_grouped_fifo(self): - """ REQUEST (CORE): release waiting requests based on grouped FIFO. """ - # set volume and deadline to 0 to check first without releasing extra requests - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=0, max_transfers=1, session=self.db_session) - - # one request with an unattached DID -> one request should be released - self.db_session.query(models.Request).delete() - self.db_session.commit() - name = generate_uuid() - add_replica(self.source_rse_id, self.scope, name, 1, self.account, session=self.db_session) - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, volume=0, deadline=0, session=self.db_session) - request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - # one request with an attached DID -> one request should be released - self.db_session.query(models.Request).delete() - self.db_session.commit() - name = generate_uuid() - dataset_name = generate_uuid() - add_replica(self.source_rse_id, self.scope, name, 1, self.account, session=self.db_session) - add_did(self.scope, dataset_name, constants.DIDType.DATASET, self.account, session=self.db_session) - attach_dids(self.scope, dataset_name, [{'name': name, 'scope': self.scope}], self.account, session=self.db_session) - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'scope': self.scope, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, volume=0, deadline=0, session=self.db_session) - request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - # five requests with different requested_at and multiple attachments per collection -> release only one request -> two requests of one collection should be released - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - name2 = generate_uuid() - name3 = generate_uuid() - name4 = generate_uuid() - name5 = generate_uuid() - dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) - dataset_2_name = generate_uuid() - add_did(self.scope, dataset_2_name, constants.DIDType.DATASET, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name5, 1, self.account, session=self.db_session) - attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}, {'name': name2, 'scope': self.scope}], self.account, session=self.db_session) - attach_dids(self.scope, dataset_2_name, [{'name': name3, 'scope': self.scope}, {'name': name4, 'scope': self.scope}], self.account, session=self.db_session) - - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'retry_count': 1, - 'rule_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2000), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'requested_at': datetime.now().replace(year=2020), - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name3, - 'requested_at': datetime.now().replace(year=2015), - 'retry_count': 1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name4, - 'requested_at': datetime.now().replace(year=2010), - 'retry_count': 1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name5, - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2018), - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, deadline=0, volume=0, session=self.db_session) - request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request_1['state'] == constants.RequestState.QUEUED - request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request_2['state'] == constants.RequestState.QUEUED - request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request_3['state'] == constants.RequestState.WAITING - request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) - assert request_4['state'] == constants.RequestState.WAITING - request_5 = get_request_by_did(self.scope, name5, self.dest_rse_id, session=self.db_session) - assert request_5['state'] == constants.RequestState.WAITING - - # with maximal volume check -> release one request -> three requests should be released because of attachments and free volume space - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - name2 = generate_uuid() - name3 = generate_uuid() - dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) - attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session) - attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session) - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=10, max_transfers=1, session=self.db_session) - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'bytes': 1, - 'scope': self.scope, - 'retry_count': 1, - 'rule_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2000), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'bytes': 2, - 'requested_at': datetime.now().replace(year=2020), - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 2, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name3, - 'bytes': 3, - 'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 3, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name4, - 'bytes': 3000, - 'requested_at': datetime.now().replace(year=2020), - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 3000, - 'md5': '', - 'adler32': '' - } - }] - - queue_requests(requests, session=self.db_session) - amount_updated_requests = release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, deadline=0, volume=10, session=self.db_session) - assert amount_updated_requests == 3 - # released because it got requested first - request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request_1['state'] == constants.RequestState.QUEUED - # released because the DID is attached to the same dataset - request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request_2['state'] == constants.RequestState.QUEUED - # released because of available volume - request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request_3['state'] == constants.RequestState.QUEUED - # still waiting because there is no free volume - request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) - assert request_4['state'] == constants.RequestState.WAITING - - # with maximal volume check -> release one request -> two requests should be released because of attachments - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - name2 = generate_uuid() - name3 = generate_uuid() - name4 = generate_uuid() - dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) - attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session) - attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session) - set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=5, max_transfers=1, session=self.db_session) - request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.all_activities, state=constants.RequestState.SUBMITTED) - request.save(session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'bytes': 1, - 'scope': self.scope, - 'retry_count': 1, - 'rule_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2000), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name2, - 'bytes': 2, - 'requested_at': datetime.now().replace(year=2020), - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 2, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name3, - 'bytes': 1, - 'requested_at': datetime.now().replace(year=2020), - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name4, - 'bytes': 1, - 'requested_at': datetime.now().replace(year=2020), - 'rule_id': generate_uuid(), - 'scope': self.scope, - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - - queue_requests(requests, session=self.db_session) - release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, deadline=0, volume=5, session=self.db_session) - # released because it got requested first - request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request_1['state'] == constants.RequestState.QUEUED - # released because the DID is attached to the same dataset - request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request_2['state'] == constants.RequestState.QUEUED - # still waiting because there is no free volume after releasing the two requests above - request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request_3['state'] == constants.RequestState.WAITING - request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) - assert request_4['state'] == constants.RequestState.WAITING - # with deadline check -> release 0 requests -> 1 request should be released nonetheless - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account) - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name2, 1, self.account) - current_hour = datetime.now().hour - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now().replace(hour=current_hour - 2), - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now(), - 'request_id': generate_uuid(), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - release_waiting_requests_grouped_fifo(self.source_rse_id, count=0, deadline=1, volume=0, session=self.db_session) - # queued because of deadline - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - # waiting because count=0 - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING +@pytest.mark.parametrize('use_preparer', ['preparer enabled', 'preparer disabled']) +def test_queue_requests_state(vo, use_preparer): + """ REQUEST (CORE): test queuing requests """ - def test_release_waiting_requests_fifo(self): - """ REQUEST (CORE): release waiting requests based on FIFO. """ - # without account and activity check - # two requests -> release one request -> request with oldest requested_at date should be released - name1 = generate_uuid() - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2018), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2020), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_fifo(self.dest_rse_id, count=1, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request2['state'] == constants.RequestState.WAITING + if use_preparer == 'preparer enabled': + use_preparer = True + elif use_preparer == 'preparer disabled': + use_preparer = False + else: + return pytest.xfail(reason=f'unknown test parameter use_preparer={use_preparer}') - # with activity and account check - # two requests -> release two request -> requests with correct account and activity should be released - self.db_session.query(models.Request).delete() - self.db_session.commit() - name1 = generate_uuid() - name2 = generate_uuid() - name3 = generate_uuid() - name4 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'account': self.account, - 'requested_at': datetime.now().replace(year=2018), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2020), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'account': self.account, - 'attributes': { - 'activity': 'ignore', - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2020), - 'name': name3, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'account': InternalAccount('jdoe', **self.vo), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2020), # requested latest but account and activity are correct - 'name': name4, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'account': self.account, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_fifo(self.dest_rse_id, count=2, account=self.account, activity=self.user_activity, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING - request = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - def test_release_waiting_requests_all(self): - """ REQUEST (CORE): release all waiting requests. """ - name1 = generate_uuid() - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - requests = [{ - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'requested_at': datetime.now().replace(year=2018), - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'dest_rse_id': self.dest_rse_id, - 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'request_id': generate_uuid(), - 'requested_at': datetime.now().replace(year=2020), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_all_waiting_requests(self.dest_rse_id, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - - @skiplimitedsql - def test_release_waiting_requests_per_deadline(self): - """ REQUEST (CORE): release grouped waiting requests that exceeded waiting time.""" - # a request that exceeded the maximal waiting time to be released (1 hour) -> release one request -> only the exceeded request should be released - set_rse_transfer_limits(self.source_rse_id, activity=self.all_activities, strategy='grouped_fifo', session=self.db_session) - name1 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - current_hour = datetime.now().hour - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now().replace(hour=current_hour - 2), - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now(), - 'request_id': generate_uuid(), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - release_waiting_requests_per_deadline(self.source_rse_id, deadline=1, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING + db_session = session.get_session() + dest_rse = 'MOCK' + dest_rse2 = 'MOCK2' + source_rse = 'MOCK4' + source_rse2 = 'MOCK5' + dest_rse_id = get_rse_id(dest_rse, vo=vo) + dest_rse_id2 = get_rse_id(dest_rse2, vo=vo) + source_rse_id = get_rse_id(source_rse, vo=vo) + source_rse_id2 = get_rse_id(source_rse2, vo=vo) + scope = InternalScope('mock', vo=vo) + account = InternalAccount('root', vo=vo) + user_activity = 'User Subscription' + config_set('conveyor', 'use_preparer', str(use_preparer)) + target_state = RequestState.PREPARING if use_preparer else RequestState.QUEUED - # a request that exceeded the maximal waiting time to be released (1 hour) -> release one request -> release all requests of the same dataset - name1 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) - name2 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) - name3 = generate_uuid() - add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) - dataset_name = generate_uuid() - add_did(self.scope, dataset_name, constants.DIDType.DATASET, self.account, session=self.db_session) - attach_dids(self.scope, dataset_name, [{'name': name1, 'scope': self.scope}, {'name': name2, 'scope': self.scope}], self.account, session=self.db_session) - current_hour = datetime.now().hour - requests = [{ - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now().replace(hour=current_hour - 2), - 'request_id': generate_uuid(), - 'name': name1, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now(), - 'request_id': generate_uuid(), - 'name': name2, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }, { - 'source_rse_id': self.source_rse_id, - 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, - 'requested_at': datetime.now(), - 'request_id': generate_uuid(), - 'name': name3, - 'scope': self.scope, - 'rule_id': generate_uuid(), - 'retry_count': 1, - 'attributes': { - 'activity': self.user_activity, - 'bytes': 1, - 'md5': '', - 'adler32': '' - } - }] - queue_requests(requests, session=self.db_session) - release_waiting_requests_per_deadline(self.source_rse_id, deadline=1, session=self.db_session) - request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.QUEUED - request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request['state'] == constants.RequestState.WAITING + name = generate_uuid() + name2 = generate_uuid() + name3 = generate_uuid() + add_replica(source_rse_id, scope, name, 1, account, session=db_session) + add_replica(source_rse_id2, scope, name2, 1, account, session=db_session) + add_replica(source_rse_id, scope, name3, 1, account, session=db_session) + + set_rse_transfer_limits(dest_rse_id, user_activity, max_transfers=1, session=db_session) + set_rse_transfer_limits(dest_rse_id2, user_activity, max_transfers=1, session=db_session) + set_rse_transfer_limits(source_rse_id, user_activity, max_transfers=1, session=db_session) + set_rse_transfer_limits(source_rse_id2, user_activity, max_transfers=1, session=db_session) + + requests = [{ + 'dest_rse_id': dest_rse_id, + 'src_rse_id': source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name, + 'scope': scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2015), + 'attributes': { + 'activity': user_activity, + 'bytes': 10, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': dest_rse_id, + 'src_rse_id': source_rse_id2, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name2, + 'scope': scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2015), + 'attributes': { + 'activity': 'unknown', + 'bytes': 10, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': dest_rse_id2, + 'src_rse_id': source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name3, + 'scope': scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2015), + 'attributes': { + 'activity': user_activity, + 'bytes': 10, + 'md5': '', + 'adler32': '' + } + }] + try: + queue_requests(requests, session=db_session) + request = get_request_by_did(scope, name, dest_rse_id, session=db_session) + assert request['state'] == target_state + request = get_request_by_did(scope, name2, dest_rse_id, session=db_session) + assert request['state'] == target_state + request = get_request_by_did(scope, name3, dest_rse_id2, session=db_session) + assert request['state'] == target_state + + finally: + config_remove_option('conveyor', 'use_preparer') + db_session.query(models.Source).delete() + db_session.query(models.Request).delete() + db_session.query(models.RSETransferLimit).delete() + db_session.query(models.Distance).delete() + db_session.query(models.Config).delete() + db_session.commit() class TestRequestCoreList(unittest.TestCase): diff --git a/lib/rucio/tests/test_throttler.py b/lib/rucio/tests/test_throttler.py index 8d86d34caf..4f29d949eb 100644 --- a/lib/rucio/tests/test_throttler.py +++ b/lib/rucio/tests/test_throttler.py @@ -1,4 +1,5 @@ -# Copyright 2019-2020 CERN for the benefit of the ATLAS collaboration. +# -*- coding: utf-8 -*- +# Copyright 2019-2020 CERN # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,29 +19,30 @@ # - Eli Chadwick , 2020 # - Patrick Austin , 2020 # - Benedikt Ziemons , 2020 -# -# PY3K COMPATIBLE import unittest from datetime import datetime -from rucio.common.config import config_get, config_get_bool +from rucio.common.config import config_get, config_get_bool, config_set, config_remove_option from rucio.common.types import InternalAccount, InternalScope from rucio.common.utils import generate_uuid from rucio.core.config import set from rucio.core.did import attach_dids, add_did from rucio.core.replica import add_replica -from rucio.core.request import queue_requests, get_request_by_did +from rucio.core.request import queue_requests, get_request_by_did, release_waiting_requests_per_deadline, release_all_waiting_requests, release_waiting_requests_fifo, release_waiting_requests_grouped_fifo, release_waiting_requests_per_free_volume from rucio.core.rse import get_rse_id, set_rse_transfer_limits -from rucio.daemons.conveyor import throttler -from rucio.db.sqla import session, models, constants -from rucio.tests.test_request import skiplimitedsql +from rucio.daemons.conveyor import throttler, preparer +from rucio.db.sqla import session, models +from rucio.db.sqla.constants import DIDType, RequestType, RequestState +from rucio.tests.common import skiplimitedsql class TestThrottlerGroupedFIFO(unittest.TestCase): """Throttler per destination RSE and on all activites per grouped FIFO """ + db_session = None + @classmethod def setUpClass(cls): if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): @@ -58,6 +60,7 @@ def setUpClass(cls): cls.account = InternalAccount('root', **cls.vo) cls.user_activity = 'User Subscription' cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') def setUp(self): self.db_session.query(models.Request).delete() @@ -68,10 +71,11 @@ def setUp(self): @classmethod def tearDownClass(cls): + config_remove_option('conveyor', 'use_preparer') cls.db_session.commit() cls.db_session.close() - def test_throttler_grouped_fifo_all(self): + def test_preparer_throttler_grouped_fifo_all(self): """ THROTTLER (CLIENTS): throttler release all waiting requests (DEST - ALL ACT - GFIFO). """ # no threshold when releasing -> release all waiting requests @@ -82,7 +86,7 @@ def test_throttler_grouped_fifo_all(self): name3 = generate_uuid() name4 = generate_uuid() dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) @@ -92,7 +96,7 @@ def test_throttler_grouped_fifo_all(self): requests = [{ 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'bytes': 1, @@ -109,7 +113,7 @@ def test_throttler_grouped_fifo_all(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name2, 'bytes': 2, @@ -126,7 +130,7 @@ def test_throttler_grouped_fifo_all(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name3, 'bytes': 3, @@ -143,7 +147,7 @@ def test_throttler_grouped_fifo_all(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name4, 'bytes': 3000, @@ -160,26 +164,30 @@ def test_throttler_grouped_fifo_all(self): }] queue_requests(requests, session=self.db_session) - self.db_session.query(models.RSETransferLimit).delete() self.db_session.commit() + + preparer.run(once=True, bulk=None) request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request_1['state'] == constants.RequestState.WAITING + assert request_1['state'] == RequestState.WAITING request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request_2['state'] == constants.RequestState.WAITING + assert request_2['state'] == RequestState.WAITING request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id) - assert request_3['state'] == constants.RequestState.WAITING + assert request_3['state'] == RequestState.WAITING request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id) - assert request_4['state'] == constants.RequestState.WAITING + assert request_4['state'] == RequestState.WAITING + + self.db_session.query(models.RSETransferLimit).delete() + self.db_session.commit() throttler.run(once=True, sleep_time=1) request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request_1['state'] == constants.RequestState.QUEUED + assert request_1['state'] == RequestState.QUEUED request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request_2['state'] == constants.RequestState.QUEUED + assert request_2['state'] == RequestState.QUEUED request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id) - assert request_3['state'] == constants.RequestState.QUEUED + assert request_3['state'] == RequestState.QUEUED request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id) - assert request_4['state'] == constants.RequestState.QUEUED + assert request_4['state'] == RequestState.QUEUED def test_throttler_grouped_fifo_nothing(self): """ THROTTLER (CLIENTS): throttler release nothing (DEST - ALL ACT - GFIFO). """ @@ -187,14 +195,14 @@ def test_throttler_grouped_fifo_nothing(self): # four waiting requests and one active requests but threshold is 1 # more than 80% of the transfer limit are already used -> release nothing set_rse_transfer_limits(self.dest_rse_id, max_transfers=1, activity=self.all_activities, strategy='grouped_fifo', session=self.db_session) - request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED) + request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=RequestState.SUBMITTED) request.save(session=self.db_session) name1 = generate_uuid() name2 = generate_uuid() name3 = generate_uuid() name4 = generate_uuid() dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) @@ -204,7 +212,7 @@ def test_throttler_grouped_fifo_nothing(self): requests = [{ 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'bytes': 1, @@ -221,7 +229,7 @@ def test_throttler_grouped_fifo_nothing(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name2, 'bytes': 2, @@ -238,7 +246,7 @@ def test_throttler_grouped_fifo_nothing(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name3, 'bytes': 3, @@ -255,7 +263,7 @@ def test_throttler_grouped_fifo_nothing(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name4, 'bytes': 3000, @@ -273,15 +281,16 @@ def test_throttler_grouped_fifo_nothing(self): queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request_1['state'] == constants.RequestState.WAITING + assert request_1['state'] == RequestState.WAITING request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request_2['state'] == constants.RequestState.WAITING + assert request_2['state'] == RequestState.WAITING request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id) - assert request_3['state'] == constants.RequestState.WAITING + assert request_3['state'] == RequestState.WAITING request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id) - assert request_4['state'] == constants.RequestState.WAITING + assert request_4['state'] == RequestState.WAITING @skiplimitedsql def test_throttler_grouped_fifo_subset(self): @@ -292,7 +301,7 @@ def test_throttler_grouped_fifo_subset(self): name3 = generate_uuid() name4 = generate_uuid() dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) @@ -302,7 +311,7 @@ def test_throttler_grouped_fifo_subset(self): requests = [{ 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'bytes': 1, @@ -319,7 +328,7 @@ def test_throttler_grouped_fifo_subset(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name2, 'bytes': 2, @@ -336,7 +345,7 @@ def test_throttler_grouped_fifo_subset(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name3, 'bytes': 3, @@ -353,7 +362,7 @@ def test_throttler_grouped_fifo_subset(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name4, 'bytes': 3000, @@ -371,24 +380,26 @@ def test_throttler_grouped_fifo_subset(self): queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) # released because it got requested first request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request_1['state'] == constants.RequestState.QUEUED + assert request_1['state'] == RequestState.QUEUED # released because the DID is attached to the same dataset request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request_2['state'] == constants.RequestState.QUEUED + assert request_2['state'] == RequestState.QUEUED # released because of available volume request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id) - assert request_3['state'] == constants.RequestState.QUEUED + assert request_3['state'] == RequestState.QUEUED # still waiting because there is no free volume request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id) - assert request_4['state'] == constants.RequestState.WAITING + assert request_4['state'] == RequestState.WAITING # deadline check should not work for destination RSEs - only for reading # Throttler per destination RSE and on each activites per FIFO class TestThrottlerFIFO(unittest.TestCase): + db_session = None @classmethod def setUpClass(cls): @@ -407,6 +418,7 @@ def setUpClass(cls): cls.account = InternalAccount('root', **cls.vo) cls.user_activity = 'User Subscription' cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') def setUp(self): self.db_session.query(models.Request).delete() @@ -417,6 +429,7 @@ def setUp(self): @classmethod def tearDownClass(cls): + config_remove_option('conveyor', 'use_preparer') cls.db_session.commit() cls.db_session.close() @@ -433,7 +446,7 @@ def test_throttler_fifo_release_all(self): requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -450,7 +463,7 @@ def test_throttler_fifo_release_all(self): }, { 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name2, @@ -468,11 +481,12 @@ def test_throttler_fifo_release_all(self): queue_requests(requests, session=self.db_session) self.db_session.query(models.RSETransferLimit).delete() self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.QUEUED + assert request['state'] == RequestState.QUEUED request2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request2['state'] == constants.RequestState.QUEUED + assert request2['state'] == RequestState.QUEUED # active transfers + waiting requests are less than the threshold -> release all waiting requests self.db_session.query(models.Request).delete() @@ -480,12 +494,12 @@ def test_throttler_fifo_release_all(self): name1 = generate_uuid() add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) set_rse_transfer_limits(self.dest_rse_id, activity=self.user_activity, max_transfers=3, strategy='fifo', session=self.db_session) - request = models.Request(dest_rse_id=self.dest_rse_id, activity=self.user_activity, state=constants.RequestState.SUBMITTED) + request = models.Request(dest_rse_id=self.dest_rse_id, activity=self.user_activity, state=RequestState.SUBMITTED) request.save(session=self.db_session) requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -502,9 +516,10 @@ def test_throttler_fifo_release_all(self): }] queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.QUEUED + assert request['state'] == RequestState.QUEUED def test_throttler_fifo_release_nothing(self): """ THROTTLER (CLIENTS): throttler release nothing (DEST - ACT - FIFO). """ @@ -514,7 +529,7 @@ def test_throttler_fifo_release_nothing(self): # two waiting requests and one active requests but threshold is 1 # more than 80% of the transfer limit are already used -> release nothing set_rse_transfer_limits(self.dest_rse_id, max_transfers=1, activity=self.user_activity, strategy='fifo', session=self.db_session) - request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=constants.RequestState.SUBMITTED) + request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.user_activity, state=RequestState.SUBMITTED) request.save(session=self.db_session) name1 = generate_uuid() name2 = generate_uuid() @@ -523,7 +538,7 @@ def test_throttler_fifo_release_nothing(self): requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -540,7 +555,7 @@ def test_throttler_fifo_release_nothing(self): }, { 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name2, @@ -557,11 +572,12 @@ def test_throttler_fifo_release_nothing(self): }] queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.WAITING + assert request['state'] == RequestState.WAITING request2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request2['state'] == constants.RequestState.WAITING + assert request2['state'] == RequestState.WAITING def test_throttler_fifo_release_subset(self): """ THROTTLER (CLIENTS): throttler release subset of waiting requests (DEST - ACT - FIFO). """ @@ -577,7 +593,7 @@ def test_throttler_fifo_release_subset(self): requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -594,7 +610,7 @@ def test_throttler_fifo_release_subset(self): }, { 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name2, @@ -611,15 +627,18 @@ def test_throttler_fifo_release_subset(self): }] queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.QUEUED + assert request['state'] == RequestState.QUEUED request2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request2['state'] == constants.RequestState.WAITING + assert request2['state'] == RequestState.WAITING -# Throttler per source RSE and on each activites per FIFO class TestThrottlerFIFOSRCACT(unittest.TestCase): + """Throttler per source RSE and on each activites per FIFO.""" + + db_session = None @classmethod def setUpClass(cls): @@ -643,6 +662,7 @@ def setUpClass(cls): cls.user_activity = 'User Subscription' cls.user_activity2 = 'User Subscription2' cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') def setUp(self): self.db_session.query(models.Request).delete() @@ -653,6 +673,7 @@ def setUp(self): @classmethod def tearDownClass(cls): + config_remove_option('conveyor', 'use_preparer') cls.db_session.commit() cls.db_session.close() @@ -674,7 +695,7 @@ def test_throttler_fifo_release_subset(self): requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -691,7 +712,7 @@ def test_throttler_fifo_release_subset(self): }, { 'dest_rse_id': self.dest_rse_id2, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name2, @@ -708,7 +729,7 @@ def test_throttler_fifo_release_subset(self): }, { 'dest_rse_id': self.dest_rse_id3, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name3, @@ -725,17 +746,20 @@ def test_throttler_fifo_release_subset(self): }] queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.QUEUED + assert request['state'] == RequestState.QUEUED request2 = get_request_by_did(self.scope, name2, self.dest_rse_id2) - assert request2['state'] == constants.RequestState.WAITING + assert request2['state'] == RequestState.WAITING request3 = get_request_by_did(self.scope, name3, self.dest_rse_id3) - assert request3['state'] == constants.RequestState.WAITING + assert request3['state'] == RequestState.WAITING -# Throttler per source RSE and on all activites per FIFO class TestThrottlerFIFOSRCALLACT(unittest.TestCase): + """Throttler per source RSE and on all activites per FIFO.""" + + db_session = None @classmethod def setUpClass(cls): @@ -754,6 +778,7 @@ def setUpClass(cls): cls.account = InternalAccount('root', **cls.vo) cls.user_activity = 'User Subscription' cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') def setUp(self): self.db_session.query(models.Request).delete() @@ -764,6 +789,7 @@ def setUp(self): @classmethod def tearDownClass(cls): + config_remove_option('conveyor', 'use_preparer') cls.db_session.commit() cls.db_session.close() @@ -781,7 +807,7 @@ def test_throttler_fifo_release_subset(self): requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -798,7 +824,7 @@ def test_throttler_fifo_release_subset(self): }, { 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name2, @@ -815,15 +841,18 @@ def test_throttler_fifo_release_subset(self): }] queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.QUEUED + assert request['state'] == RequestState.QUEUED request2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request2['state'] == constants.RequestState.WAITING + assert request2['state'] == RequestState.WAITING -# Throttler per destination RSE and on all activites per FIFO class TestThrottlerFIFODESTALLACT(unittest.TestCase): + """Throttler per destination RSE and on all activites per FIFO.""" + + db_session = None @classmethod def setUpClass(cls): @@ -847,6 +876,7 @@ def setUpClass(cls): cls.user_activity = 'User Subscription' cls.user_activity2 = 'User Subscription2' cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') def setUp(self): self.db_session.query(models.Request).delete() @@ -857,6 +887,7 @@ def setUp(self): @classmethod def tearDownClass(cls): + config_remove_option('conveyor', 'use_preparer') cls.db_session.commit() cls.db_session.close() @@ -878,7 +909,7 @@ def test_throttler_fifo_release_subset(self): requests = [{ 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'account': self.account, @@ -895,7 +926,7 @@ def test_throttler_fifo_release_subset(self): }, { 'dest_rse_id': self.dest_rse_id, 'source_rse_id': self.source_rse_id2, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name2, @@ -912,7 +943,7 @@ def test_throttler_fifo_release_subset(self): }, { 'dest_rse_id': self.dest_rse_id2, 'source_rse_id': self.source_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'requested_at': datetime.now().replace(year=2020), 'name': name3, @@ -929,19 +960,22 @@ def test_throttler_fifo_release_subset(self): }] queue_requests(requests, session=self.db_session) self.db_session.commit() + preparer.run(once=True, bulk=None) throttler.run(once=True, sleep_time=1) # release because max_transfers=1 request = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request['state'] == constants.RequestState.QUEUED + assert request['state'] == RequestState.QUEUED # waiting because limit already exceeded request2 = get_request_by_did(self.scope, name2, self.dest_rse_id) - assert request2['state'] == constants.RequestState.WAITING + assert request2['state'] == RequestState.WAITING request3 = get_request_by_did(self.scope, name3, self.dest_rse_id2) - assert request3['state'] == constants.RequestState.WAITING + assert request3['state'] == RequestState.WAITING -# Throttler per source RSE and on all activites per grouped FIFO class TestThrottlerGroupedFIFOSRCALLACT(unittest.TestCase): + """Throttler per source RSE and on all activites per grouped FIFO.""" + + db_session = None @classmethod def setUpClass(cls): @@ -962,6 +996,7 @@ def setUpClass(cls): cls.account = InternalAccount('root', **cls.vo) cls.user_activity = 'User Subscription' cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') def setUp(self): self.db_session.query(models.Request).delete() @@ -972,11 +1007,12 @@ def setUp(self): @classmethod def tearDownClass(cls): + config_remove_option('conveyor', 'use_preparer') cls.db_session.commit() cls.db_session.close() @skiplimitedsql - def test_throttler_grouped_fifo_subset(self): + def test_preparer_throttler_grouped_fifo_subset(self): """ THROTTLER (CLIENTS): throttler release subset of waiting requests (SRC - ALL ACT - GFIFO). """ if self.dialect == 'mysql': return True @@ -987,7 +1023,7 @@ def test_throttler_grouped_fifo_subset(self): name3 = generate_uuid() name4 = generate_uuid() dataset_1_name = generate_uuid() - add_did(self.scope, dataset_1_name, constants.DIDType.DATASET, self.account, session=self.db_session) + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) @@ -997,7 +1033,7 @@ def test_throttler_grouped_fifo_subset(self): requests = [{ 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name1, 'bytes': 1, @@ -1014,7 +1050,7 @@ def test_throttler_grouped_fifo_subset(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id_2, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name2, 'bytes': 2, @@ -1031,7 +1067,7 @@ def test_throttler_grouped_fifo_subset(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name3, 'bytes': 3, @@ -1048,7 +1084,7 @@ def test_throttler_grouped_fifo_subset(self): }, { 'source_rse_id': self.source_rse_id, 'dest_rse_id': self.dest_rse_id_2, - 'request_type': constants.RequestType.TRANSFER, + 'request_type': RequestType.TRANSFER, 'request_id': generate_uuid(), 'name': name4, 'bytes': 3000, @@ -1065,26 +1101,1038 @@ def test_throttler_grouped_fifo_subset(self): }] queue_requests(requests, session=self.db_session) + self.db_session.commit() + + preparer.run(once=True, bulk=None) request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) - assert request_1['state'] == constants.RequestState.WAITING + assert request_1['state'] == RequestState.WAITING request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id_2, session=self.db_session) - assert request_2['state'] == constants.RequestState.WAITING + assert request_2['state'] == RequestState.WAITING request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) - assert request_3['state'] == constants.RequestState.WAITING + assert request_3['state'] == RequestState.WAITING request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id_2, session=self.db_session) - assert request_4['state'] == constants.RequestState.WAITING + assert request_4['state'] == RequestState.WAITING - self.db_session.commit() throttler.run(once=True, sleep_time=1) - # released because it got requested first request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id) - assert request_1['state'] == constants.RequestState.QUEUED + assert request_1['state'] == RequestState.QUEUED # released because the DID is attached to the same dataset request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id_2) - assert request_2['state'] == constants.RequestState.QUEUED + assert request_2['state'] == RequestState.QUEUED # still waiting, volume check is only working for destination RSEs (writing) request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id) - assert request_3['state'] == constants.RequestState.WAITING + assert request_3['state'] == RequestState.WAITING request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id_2) - assert request_4['state'] == constants.RequestState.WAITING + assert request_4['state'] == RequestState.WAITING + + +class TestRequestCoreRelease(unittest.TestCase): + """Test release methods used in throttler.""" + + db_session = None + + @classmethod + def setUpClass(cls): + if config_get_bool('common', 'multi_vo', raise_exception=False, default=False): + cls.vo = {'vo': config_get('client', 'vo', raise_exception=False, default='tst')} + else: + cls.vo = {} + + cls.db_session = session.get_session() + cls.dialect = cls.db_session.bind.dialect.name + cls.dest_rse = 'MOCK' + cls.source_rse = 'MOCK4' + cls.source_rse2 = 'MOCK5' + cls.dest_rse_id = get_rse_id(cls.dest_rse, **cls.vo) + cls.source_rse_id = get_rse_id(cls.source_rse, **cls.vo) + cls.source_rse_id2 = get_rse_id(cls.source_rse2, **cls.vo) + cls.scope = InternalScope('mock', **cls.vo) + cls.account = InternalAccount('root', **cls.vo) + cls.user_activity = 'User Subscription' + cls.all_activities = 'all_activities' + config_set('conveyor', 'use_preparer', 'true') + + def setUp(self): + self.db_session.query(models.Request).delete() + self.db_session.query(models.RSETransferLimit).delete() + self.db_session.query(models.Distance).delete() + self.db_session.query(models.Config).delete() + # set transfer limits to put requests in waiting state + set_rse_transfer_limits(self.dest_rse_id, self.user_activity, max_transfers=1, session=self.db_session) + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, max_transfers=1, session=self.db_session) + set_rse_transfer_limits(self.dest_rse_id, 'ignore', max_transfers=1, session=self.db_session) + set('throttler', 'mode', 'DEST_PER_ACT', session=self.db_session) + self.db_session.commit() + + def tearDown(self): + self.db_session.commit() + + @classmethod + def tearDownClass(cls) -> None: + config_remove_option('conveyor', 'use_preparer') + + @skiplimitedsql + def test_release_waiting_requests_per_free_volume(self): + """ REQUEST (CORE): release waiting requests that fit grouped in available volume.""" + # release unattached requests that fit in available volume with respect to already submitted transfers + name1 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + name3 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.all_activities, state=RequestState.SUBMITTED) + request.save(session=self.db_session) + volume = 10 + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=volume, max_transfers=1, session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2015), + 'attributes': { + 'activity': 'User Subscription', + 'bytes': 8, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name2, + 'requested_at': datetime.now().replace(year=2020), + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': 'User Subscription', + 'bytes': 2, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name3, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2000), + 'attributes': { + 'activity': 'User Subscription', + 'bytes': 10, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_per_free_volume(self.dest_rse_id, volume=volume, session=self.db_session) + # released because small enough + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + # still waiting because requested later and to big + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + # still waiting because too big + request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + + # release attached requests that fit together with the dataset in available volume with respect to already submitted transfers + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + name3 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + name4 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) + dataset1_name = generate_uuid() + add_did(self.scope, dataset1_name, DIDType.DATASET, self.account, session=self.db_session) + attach_dids(self.scope, dataset1_name, [{'name': name1, 'scope': self.scope}, {'name': name4, 'scope': self.scope}], self.account, session=self.db_session) + dataset2_name = generate_uuid() + add_did(self.scope, dataset2_name, DIDType.DATASET, self.account, session=self.db_session) + attach_dids(self.scope, dataset2_name, [{'name': name2, 'scope': self.scope}, {'name': name3, 'scope': self.scope}], self.account, session=self.db_session) + request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.all_activities, state=RequestState.SUBMITTED) + request.save(session=self.db_session) + volume = 10 + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=volume, max_transfers=1, session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2015), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 6, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name2, + 'requested_at': datetime.now().replace(year=2020), + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 2, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name3, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2000), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 10, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name4, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2030), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 2, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_per_free_volume(self.dest_rse_id, volume=volume, session=self.db_session) + # released because dataset fits in volume + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + # waiting because dataset is too big + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + + # release requests with no available volume -> release nothing + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + add_replica(self.dest_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + volume = 0 + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=volume, max_transfers=1, session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2015), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 8, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_per_free_volume(self.dest_rse_id, volume=volume, session=self.db_session) + # waiting because no available volume + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + + @skiplimitedsql + def test_release_waiting_requests_grouped_fifo(self): + """ REQUEST (CORE): release waiting requests based on grouped FIFO. """ + # set volume and deadline to 0 to check first without releasing extra requests + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=0, max_transfers=1, session=self.db_session) + + # one request with an unattached DID -> one request should be released + self.db_session.query(models.Request).delete() + self.db_session.commit() + name = generate_uuid() + add_replica(self.source_rse_id, self.scope, name, 1, self.account, session=self.db_session) + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, volume=0, deadline=0, session=self.db_session) + request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + + # one request with an attached DID -> one request should be released + self.db_session.query(models.Request).delete() + self.db_session.commit() + name = generate_uuid() + dataset_name = generate_uuid() + add_replica(self.source_rse_id, self.scope, name, 1, self.account, session=self.db_session) + add_did(self.scope, dataset_name, DIDType.DATASET, self.account, session=self.db_session) + attach_dids(self.scope, dataset_name, [{'name': name, 'scope': self.scope}], self.account, session=self.db_session) + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'scope': self.scope, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, volume=0, deadline=0, session=self.db_session) + request = get_request_by_did(self.scope, name, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + + # five requests with different requested_at and multiple attachments per collection -> release only one request -> two requests of one collection should be released + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + name2 = generate_uuid() + name3 = generate_uuid() + name4 = generate_uuid() + name5 = generate_uuid() + dataset_1_name = generate_uuid() + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) + dataset_2_name = generate_uuid() + add_did(self.scope, dataset_2_name, DIDType.DATASET, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name5, 1, self.account, session=self.db_session) + attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}, {'name': name2, 'scope': self.scope}], self.account, session=self.db_session) + attach_dids(self.scope, dataset_2_name, [{'name': name3, 'scope': self.scope}, {'name': name4, 'scope': self.scope}], self.account, session=self.db_session) + + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'retry_count': 1, + 'rule_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2000), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name2, + 'requested_at': datetime.now().replace(year=2020), + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name3, + 'requested_at': datetime.now().replace(year=2015), + 'retry_count': 1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name4, + 'requested_at': datetime.now().replace(year=2010), + 'retry_count': 1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name5, + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2018), + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, deadline=0, volume=0, session=self.db_session) + request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request_1['state'] == RequestState.QUEUED + request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request_2['state'] == RequestState.QUEUED + request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request_3['state'] == RequestState.WAITING + request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) + assert request_4['state'] == RequestState.WAITING + request_5 = get_request_by_did(self.scope, name5, self.dest_rse_id, session=self.db_session) + assert request_5['state'] == RequestState.WAITING + + # with maximal volume check -> release one request -> three requests should be released because of attachments and free volume space + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + name2 = generate_uuid() + name3 = generate_uuid() + dataset_1_name = generate_uuid() + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) + attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session) + attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session) + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=10, max_transfers=1, session=self.db_session) + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'bytes': 1, + 'scope': self.scope, + 'retry_count': 1, + 'rule_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2000), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name2, + 'bytes': 2, + 'requested_at': datetime.now().replace(year=2020), + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 2, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name3, + 'bytes': 3, + 'requested_at': datetime.now().replace(year=2021), # requested after the request below but small enough for max_volume check + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 3, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name4, + 'bytes': 3000, + 'requested_at': datetime.now().replace(year=2020), + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 3000, + 'md5': '', + 'adler32': '' + } + }] + + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + amount_updated_requests = release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, deadline=0, volume=10, session=self.db_session) + assert amount_updated_requests == 3 + # released because it got requested first + request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request_1['state'] == RequestState.QUEUED + # released because the DID is attached to the same dataset + request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request_2['state'] == RequestState.QUEUED + # released because of available volume + request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request_3['state'] == RequestState.QUEUED + # still waiting because there is no free volume + request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) + assert request_4['state'] == RequestState.WAITING + + # with maximal volume check -> release one request -> two requests should be released because of attachments + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + name2 = generate_uuid() + name3 = generate_uuid() + name4 = generate_uuid() + dataset_1_name = generate_uuid() + add_did(self.scope, dataset_1_name, DIDType.DATASET, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) + attach_dids(self.scope, dataset_1_name, [{'name': name1, 'scope': self.scope}], self.account, session=self.db_session) + attach_dids(self.scope, dataset_1_name, [{'name': name2, 'scope': self.scope}], self.account, session=self.db_session) + set_rse_transfer_limits(self.dest_rse_id, self.all_activities, volume=5, max_transfers=1, session=self.db_session) + request = models.Request(dest_rse_id=self.dest_rse_id, bytes=2, activity=self.all_activities, state=RequestState.SUBMITTED) + request.save(session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'bytes': 1, + 'scope': self.scope, + 'retry_count': 1, + 'rule_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2000), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name2, + 'bytes': 2, + 'requested_at': datetime.now().replace(year=2020), + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 2, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name3, + 'bytes': 1, + 'requested_at': datetime.now().replace(year=2020), + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name4, + 'bytes': 1, + 'requested_at': datetime.now().replace(year=2020), + 'rule_id': generate_uuid(), + 'scope': self.scope, + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_grouped_fifo(self.dest_rse_id, count=1, deadline=0, volume=5, session=self.db_session) + # released because it got requested first + request_1 = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request_1['state'] == RequestState.QUEUED + # released because the DID is attached to the same dataset + request_2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request_2['state'] == RequestState.QUEUED + # still waiting because there is no free volume after releasing the two requests above + request_3 = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request_3['state'] == RequestState.WAITING + request_4 = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) + assert request_4['state'] == RequestState.WAITING + + # with deadline check -> release 0 requests -> 1 request should be released nonetheless + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account) + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name2, 1, self.account) + current_hour = datetime.now().hour + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now().replace(hour=current_hour - 2), + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now(), + 'request_id': generate_uuid(), + 'name': name2, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_grouped_fifo(self.source_rse_id, count=0, deadline=1, volume=0, session=self.db_session) + # queued because of deadline + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + # waiting because count=0 + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + + def test_release_waiting_requests_fifo(self): + """ REQUEST (CORE): release waiting requests based on FIFO. """ + # without account and activity check + # two requests -> release one request -> request with oldest requested_at date should be released + name1 = generate_uuid() + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2018), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2020), + 'name': name2, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_fifo(self.dest_rse_id, count=1, session=self.db_session) + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request2 = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request2['state'] == RequestState.WAITING + + # with activity and account check + # two requests -> release two request -> requests with correct account and activity should be released + self.db_session.query(models.Request).delete() + self.db_session.commit() + name1 = generate_uuid() + name2 = generate_uuid() + name3 = generate_uuid() + name4 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name4, 1, self.account, session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'account': self.account, + 'requested_at': datetime.now().replace(year=2018), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2020), + 'name': name2, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'account': self.account, + 'attributes': { + 'activity': 'ignore', + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2020), + 'name': name3, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'account': InternalAccount('jdoe', **self.vo), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2020), # requested latest but account and activity are correct + 'name': name4, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'account': self.account, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_fifo(self.dest_rse_id, count=2, account=self.account, activity=self.user_activity, session=self.db_session) + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + request = get_request_by_did(self.scope, name4, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + + def test_release_waiting_requests_all(self): + """ REQUEST (CORE): release all waiting requests. """ + name1 = generate_uuid() + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + requests = [{ + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'requested_at': datetime.now().replace(year=2018), + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'dest_rse_id': self.dest_rse_id, + 'source_rse_id': self.source_rse_id, + 'request_type': RequestType.TRANSFER, + 'request_id': generate_uuid(), + 'requested_at': datetime.now().replace(year=2020), + 'name': name2, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_all_waiting_requests(self.dest_rse_id, session=self.db_session) + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + + @skiplimitedsql + def test_release_waiting_requests_per_deadline(self): + """ REQUEST (CORE): release grouped waiting requests that exceeded waiting time.""" + # a request that exceeded the maximal waiting time to be released (1 hour) -> release one request -> only the exceeded request should be released + set_rse_transfer_limits(self.source_rse_id, activity=self.all_activities, strategy='grouped_fifo', session=self.db_session) + name1 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + current_hour = datetime.now().hour + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now().replace(hour=current_hour - 2), + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now(), + 'request_id': generate_uuid(), + 'name': name2, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_per_deadline(self.source_rse_id, deadline=1, session=self.db_session) + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING + + # a request that exceeded the maximal waiting time to be released (1 hour) -> release one request -> release all requests of the same dataset + name1 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name1, 1, self.account, session=self.db_session) + name2 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name2, 1, self.account, session=self.db_session) + name3 = generate_uuid() + add_replica(self.source_rse_id, self.scope, name3, 1, self.account, session=self.db_session) + dataset_name = generate_uuid() + add_did(self.scope, dataset_name, DIDType.DATASET, self.account, session=self.db_session) + attach_dids(self.scope, dataset_name, [{'name': name1, 'scope': self.scope}, {'name': name2, 'scope': self.scope}], self.account, session=self.db_session) + current_hour = datetime.now().hour + requests = [{ + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now().replace(hour=current_hour - 2), + 'request_id': generate_uuid(), + 'name': name1, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now(), + 'request_id': generate_uuid(), + 'name': name2, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }, { + 'source_rse_id': self.source_rse_id, + 'dest_rse_id': self.dest_rse_id, + 'request_type': RequestType.TRANSFER, + 'requested_at': datetime.now(), + 'request_id': generate_uuid(), + 'name': name3, + 'scope': self.scope, + 'rule_id': generate_uuid(), + 'retry_count': 1, + 'attributes': { + 'activity': self.user_activity, + 'bytes': 1, + 'md5': '', + 'adler32': '' + } + }] + queue_requests(requests, session=self.db_session) + preparer.run_once(session=self.db_session) + self.db_session.commit() + release_waiting_requests_per_deadline(self.source_rse_id, deadline=1, session=self.db_session) + request = get_request_by_did(self.scope, name1, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request = get_request_by_did(self.scope, name2, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.QUEUED + request = get_request_by_did(self.scope, name3, self.dest_rse_id, session=self.db_session) + assert request['state'] == RequestState.WAITING diff --git a/tools/add_header b/tools/add_header index 13a97a2a74..6f875b0661 100755 --- a/tools/add_header +++ b/tools/add_header @@ -193,6 +193,7 @@ def main(arguments): header += '# - %(name)s %(mail)s, %(min)s\n' % authors[author] else: header += '# - %(name)s %(mail)s, %(min)s-%(max)s\n' % authors[author] + header += '\n' if not arguments.dry_run: if not arguments.in_place: @@ -217,16 +218,12 @@ def main(arguments): modified.write(lines[0]) modified.write(header) - first = True count = 50 # only deletes old comments in the first n lines or until the first non-comment for line in lines: not_comment_line = not line.startswith('#') and len(line.lstrip()) != 0 if count != 0 and not_comment_line: count = 0 if count == 0 or not_comment_line: - if first: - first = False - modified.write('\n') modified.write(line) else: