Core & Internals: Add is_old_db method; Fix rucio#1157

… to test for an old alembic revision in the database.
Move alembic migration to tools/alembic_migration.sh.
Add is_old_db test in alembic_migration.sh.
Add startup tests for old DB in all daemons.
Add test_daemons.py for testing all daemons on that behavior.

lib/rucio/alembicrevision.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+
+ALEMBIC_REVISION = '50280c53117c'  # the current alembic head revision

lib/rucio/daemons/abacus/account.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2014-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,11 +14,12 @@
 # limitations under the License.
 #
 # Authors:
-# - Martin Barisits <martin.barisits@cern.ch>, 2014-2016
-# - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2014-2019
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -33,10 +35,12 @@
 import time
 import traceback
 
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.utils import get_thread_with_periodic_running_function
 from rucio.core.heartbeat import live, die, sanity_check
 from rucio.core.account_counter import get_updated_account_counters, update_account_counter, fill_account_counter_history_table
+from rucio.db.sqla.util import is_old_db
 
 graceful_stop = threading.Event()
 
@@ -109,6 +113,9 @@ def run(once=False, threads=1, fill_history_table=False):
     """
     Starts up the Abacus-Account threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     executable = 'abacus-account'
     hostname = socket.gethostname()
     sanity_check(executable=executable, hostname=hostname)

lib/rucio/daemons/abacus/collection_replica.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +14,9 @@
 # limitations under the License.
 #
 # Authors:
-# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 
 """
 Abacus-Collection-Replica is a daemon to update collection replica.
@@ -28,9 +30,11 @@
 import time
 import traceback
 
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.core.heartbeat import live, die, sanity_check
 from rucio.core.replica import get_cleaned_updated_collection_replicas, update_collection_replica
+from rucio.db.sqla.util import is_old_db
 
 graceful_stop = threading.Event()
 
@@ -102,6 +106,9 @@ def run(once=False, threads=1):
     """
     Starts up the Abacus-Collection-Replica threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     executable = 'abacus-collection-replica'
     hostname = socket.gethostname()
     sanity_check(executable=executable, hostname=hostname)

lib/rucio/daemons/abacus/rse.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2014-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,10 +15,11 @@
 #
 # Authors:
 # - Martin Barisits <martin.barisits@cern.ch>, 2014-2016
-# - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -33,10 +35,12 @@
 import time
 import traceback
 
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.utils import get_thread_with_periodic_running_function
 from rucio.core.heartbeat import live, die, sanity_check
 from rucio.core.rse_counter import get_updated_rse_counters, update_rse_counter, fill_rse_counter_history_table
+from rucio.db.sqla.util import is_old_db
 
 graceful_stop = threading.Event()
 
@@ -108,6 +112,9 @@ def run(once=False, threads=1, fill_history_table=False):
     """
     Starts up the Abacus-RSE threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     executable = 'abacus-rse'
     hostname = socket.gethostname()
     sanity_check(executable=executable, hostname=hostname)

lib/rucio/daemons/atropos/atropos.py

@@ -1,4 +1,5 @@
-# Copyright 2016-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,14 +14,15 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2018
-# - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
 # - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
-# - Patrick Austin, <patrick.austin@stfc.ac.uk>, 2020
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -35,6 +37,7 @@
 from sys import exc_info, stdout
 from traceback import format_exception
 
+from rucio.common import exception
 from rucio.db.sqla.constants import LifetimeExceptionsState
 from rucio.common.config import config_get
 from rucio.common.exception import InvalidRSEExpression, RuleNotFound
@@ -44,6 +47,7 @@
 from rucio.core.rse import get_rse_name, get_rse_vo
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.rule import get_rules_beyond_eol, update_rule
+from rucio.db.sqla.util import is_old_db
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -203,6 +207,9 @@ def run(threads=1, bulk=100, date_check=None, dry_run=True, grace_period=86400,
     """
     Starts up the atropos threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     if not date_check:
         date_check = datetime.datetime.now()
     else:

lib/rucio/daemons/automatix/automatix.py

@@ -1,4 +1,5 @@
-# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,16 +14,15 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
-# - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2014
-# - Tomas Kouba <tomas.kouba@cern.ch>, 2015
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2020
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -44,10 +44,12 @@
 from time import sleep, time
 
 from rucio.client import Client
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.utils import adler32
 from rucio.core.config import get
 from rucio.core import monitor, heartbeat
+from rucio.db.sqla.util import is_old_db
 from rucio.rse import rsemanager as rsemgr
 from rucio.core.scope import list_scopes
 from rucio.common.types import InternalScope
@@ -300,6 +302,9 @@ def run(total_workers=1, once=False, inputfile=None):
     """
     Starts up the automatix threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     try:
         sites = [s.strip() for s in get('automatix', 'sites').split(',')]
     except Exception:

lib/rucio/daemons/badreplicas/minos.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -37,6 +38,7 @@
 from datetime import datetime
 from sys import stdout
 
+from rucio.common import exception
 from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, ReplicaState
 
 from rucio.db.sqla.session import get_session
@@ -50,7 +52,7 @@
 from rucio.core.rse import get_rse_name
 
 from rucio.core import heartbeat
-
+from rucio.db.sqla.util import is_old_db
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -271,6 +273,8 @@ def run(threads=1, bulk=100, once=False, sleep_time=60):
     """
     Starts up the minos threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
     if once:
         logging.info('Will run only one iteration in a single threaded mode')

lib/rucio/daemons/badreplicas/minos_temporary_expiration.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,8 +14,8 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
 # - Martin Barisits <martin.barisits@cern.ch>, 2018-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
@@ -32,6 +33,7 @@
 
 from sys import stdout
 
+from rucio.common import exception
 from rucio.db.sqla.constants import BadFilesStatus, ReplicaState
 
 from rucio.db.sqla.session import get_session
@@ -43,7 +45,7 @@
                                 bulk_delete_bad_replicas, list_expired_temporary_unavailable_replicas)
 
 from rucio.core import heartbeat
-
+from rucio.db.sqla.util import is_old_db
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -163,6 +165,8 @@ def run(threads=1, bulk=100, once=False, sleep_time=60):
     """
     Starts up the minos threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
     if once:
         logging.info('Will run only one iteration in a single threaded mode')

lib/rucio/daemons/badreplicas/necromancer.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +14,11 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
-# - Vincent Garonne <vgaronne@gmail.com>, 2015-2018
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2015
-# - Wen Guan <wguan.icedew@gmail.com>, 2015
-# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2019
+# - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -35,14 +34,15 @@
 from sys import exc_info, stdout
 from traceback import format_exception
 
+from rucio.common import exception
 from rucio.db.sqla.constants import ReplicaState
 from rucio.common.config import config_get
 from rucio.common.utils import chunks
 from rucio.common.exception import DatabaseException
 from rucio.core import monitor, heartbeat
 from rucio.core.replica import list_bad_replicas, get_replicas_state, list_bad_replicas_history, update_bad_replicas_history
 from rucio.core.rule import update_rules_for_lost_replica, update_rules_for_bad_replica
-
+from rucio.db.sqla.util import is_old_db
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -150,6 +150,8 @@ def run(threads=1, bulk=100, once=False):
     """
     Starts up the necromancer threads.
     """
+    if is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
     if once:
         logging.info('Will run only one iteration in a single threaded mode')