Core & Internals: Add is_old_db method; Fix rucio#1157

… to test for an old alembic revision in the database.
Move alembic migration to tools/alembic_migration.sh.
Add is_old_db test in alembic_migration.sh.
Add startup tests for old DB in all daemons.
Add test_daemons.py for testing all daemons on that behavior.

lib/rucio/alembicrevision.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 CERN
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+
+ALEMBIC_REVISION = '50280c53117c'  # the current alembic head revision

lib/rucio/daemons/abacus/account.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2014-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,11 +14,12 @@
 # limitations under the License.
 #
 # Authors:
-# - Martin Barisits <martin.barisits@cern.ch>, 2014-2016
-# - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2014-2019
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2014-2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -33,10 +35,12 @@
 import time
 import traceback
 
+import rucio.db.sqla.util
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.utils import get_thread_with_periodic_running_function
-from rucio.core.heartbeat import live, die, sanity_check
 from rucio.core.account_counter import get_updated_account_counters, update_account_counter, fill_account_counter_history_table
+from rucio.core.heartbeat import live, die, sanity_check
 
 graceful_stop = threading.Event()
 
@@ -109,6 +113,9 @@ def run(once=False, threads=1, fill_history_table=False):
     """
     Starts up the Abacus-Account threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     executable = 'abacus-account'
     hostname = socket.gethostname()
     sanity_check(executable=executable, hostname=hostname)

lib/rucio/daemons/abacus/collection_replica.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +14,9 @@
 # limitations under the License.
 #
 # Authors:
-# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 
 """
 Abacus-Collection-Replica is a daemon to update collection replica.
@@ -28,6 +30,8 @@
 import time
 import traceback
 
+import rucio.db.sqla.util
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.core.heartbeat import live, die, sanity_check
 from rucio.core.replica import get_cleaned_updated_collection_replicas, update_collection_replica
@@ -102,6 +106,9 @@ def run(once=False, threads=1):
     """
     Starts up the Abacus-Collection-Replica threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     executable = 'abacus-collection-replica'
     hostname = socket.gethostname()
     sanity_check(executable=executable, hostname=hostname)

lib/rucio/daemons/abacus/rse.py

@@ -1,4 +1,5 @@
-# Copyright 2014-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2014-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,10 +15,11 @@
 #
 # Authors:
 # - Martin Barisits <martin.barisits@cern.ch>, 2014-2016
-# - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -33,6 +35,8 @@
 import time
 import traceback
 
+import rucio.db.sqla.util
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.utils import get_thread_with_periodic_running_function
 from rucio.core.heartbeat import live, die, sanity_check
@@ -108,6 +112,9 @@ def run(once=False, threads=1, fill_history_table=False):
     """
     Starts up the Abacus-RSE threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     executable = 'abacus-rse'
     hostname = socket.gethostname()
     sanity_check(executable=executable, hostname=hostname)

lib/rucio/daemons/atropos/atropos.py

@@ -1,4 +1,5 @@
-# Copyright 2016-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,14 +14,15 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2016-2018
-# - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
 # - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
-# - Brandon White <bjwhite@fnal.gov>, 2019-2020
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
-# - Patrick Austin, <patrick.austin@stfc.ac.uk>, 2020
+# - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -31,19 +33,20 @@
 import socket
 import threading
 import time
-
 from sys import exc_info, stdout
 from traceback import format_exception
 
-from rucio.db.sqla.constants import LifetimeExceptionsState
+import rucio.core.lifetime_exception
+import rucio.db.sqla.util
+from rucio.common import exception
 from rucio.common.config import config_get
 from rucio.common.exception import InvalidRSEExpression, RuleNotFound
 from rucio.core import heartbeat
-import rucio.core.lifetime_exception
 from rucio.core.lock import get_dataset_locks
 from rucio.core.rse import get_rse_name, get_rse_vo
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.rule import get_rules_beyond_eol, update_rule
+from rucio.db.sqla.constants import LifetimeExceptionsState
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -203,6 +206,9 @@ def run(threads=1, bulk=100, date_check=None, dry_run=True, grace_period=86400,
     """
     Starts up the atropos threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     if not date_check:
         date_check = datetime.datetime.now()
     else:

lib/rucio/daemons/automatix/automatix.py

@@ -1,4 +1,5 @@
-# Copyright 2013-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,48 +14,46 @@
 # limitations under the License.
 #
 # Authors:
-# - Vincent Garonne <vgaronne@gmail.com>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
-# - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2014
-# - Tomas Kouba <tomas.kouba@cern.ch>, 2015
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Brandon White <bjwhite@fnal.gov>, 2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2020
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
 from __future__ import division
 from __future__ import print_function
 
 import logging
-import socket
 import random
+import socket
 import tempfile
 import threading
 import traceback
-
 from datetime import datetime
 from json import load
 from math import exp
 from os import remove, rmdir, stat, getpid
 from sys import stdout
 from time import sleep, time
 
+import rucio.db.sqla.util
 from rucio.client import Client
+from rucio.common import exception
 from rucio.common.config import config_get
+from rucio.common.exception import FileReplicaAlreadyExists, ConfigNotFound
+from rucio.common.types import InternalScope
 from rucio.common.utils import adler32
-from rucio.core.config import get
+from rucio.common.utils import execute, generate_uuid
 from rucio.core import monitor, heartbeat
-from rucio.rse import rsemanager as rsemgr
+from rucio.core.config import get
 from rucio.core.scope import list_scopes
-from rucio.common.types import InternalScope
-
-from rucio.common.utils import execute, generate_uuid
-from rucio.common.exception import FileReplicaAlreadyExists, ConfigNotFound
-
+from rucio.rse import rsemanager as rsemgr
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -300,6 +299,9 @@ def run(total_workers=1, once=False, inputfile=None):
     """
     Starts up the automatix threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
+
     try:
         sites = [s.strip() for s in get('automatix', 'sites').split(',')]
     except Exception:

lib/rucio/daemons/badreplicas/minos.py

@@ -1,4 +1,5 @@
-# Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,27 +31,25 @@
 import math
 import os
 import socket
-import traceback
 import threading
 import time
-
+import traceback
 from datetime import datetime
 from sys import stdout
 
-from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, ReplicaState
-
-from rucio.db.sqla.session import get_session
+import rucio.db.sqla.util
+from rucio.common import exception
 from rucio.common.config import config_get
-from rucio.common.utils import chunks
 from rucio.common.exception import UnsupportedOperation, DataIdentifierNotFound, ReplicaNotFound
+from rucio.common.utils import chunks
+from rucio.core import heartbeat
 from rucio.core.did import get_metadata
 from rucio.core.replica import (get_bad_pfns, get_pfn_to_rse, declare_bad_file_replicas,
                                 get_did_from_pfns, update_replicas_states, bulk_add_bad_replicas,
                                 bulk_delete_bad_pfns, get_replicas_state)
 from rucio.core.rse import get_rse_name
-
-from rucio.core import heartbeat
-
+from rucio.db.sqla.constants import BadFilesStatus, BadPFNStatus, ReplicaState
+from rucio.db.sqla.session import get_session
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -271,6 +270,8 @@ def run(threads=1, bulk=100, once=False, sleep_time=60):
     """
     Starts up the minos threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
     if once:
         logging.info('Will run only one iteration in a single threaded mode')

lib/rucio/daemons/badreplicas/minos_temporary_expiration.py

@@ -1,4 +1,5 @@
-# Copyright 2018-2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +14,8 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2019
 # - Martin Barisits <martin.barisits@cern.ch>, 2018-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Brandon White <bjwhite@fnal.gov>, 2019
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2020
@@ -27,23 +28,21 @@
 import os
 import socket
 import threading
-import traceback
 import time
-
+import traceback
 from sys import stdout
 
-from rucio.db.sqla.constants import BadFilesStatus, ReplicaState
-
-from rucio.db.sqla.session import get_session
+import rucio.db.sqla.util
+from rucio.common import exception
 from rucio.common.config import config_get
-from rucio.common.utils import chunks
 from rucio.common.exception import DataIdentifierNotFound, ReplicaNotFound
+from rucio.common.utils import chunks
+from rucio.core import heartbeat
 from rucio.core.did import get_metadata
 from rucio.core.replica import (update_replicas_states,
                                 bulk_delete_bad_replicas, list_expired_temporary_unavailable_replicas)
-
-from rucio.core import heartbeat
-
+from rucio.db.sqla.constants import BadFilesStatus, ReplicaState
+from rucio.db.sqla.session import get_session
 
 logging.basicConfig(stream=stdout,
                     level=getattr(logging,
@@ -163,6 +162,8 @@ def run(threads=1, bulk=100, once=False, sleep_time=60):
     """
     Starts up the minos threads.
     """
+    if rucio.db.sqla.util.is_old_db():
+        raise exception.DatabaseException('Database was not updated, daemon won\'t start')
 
     if once:
         logging.info('Will run only one iteration in a single threaded mode')