Upgrade SQLAlchemy from 1.3.7 to 1.3.20; Fix rucio#4055

1.3.8 had a conceptual change in the ENUM handling on PostgreSQL.
mlassnig · Nov 23, 2020 · ff919d5 · ff919d5
1 parent 51dd4f0
commit ff919d5
Show file tree

Hide file tree

Showing 53 changed files with 728 additions and 834 deletions.
diff --git a/etc/pip-requires b/etc/pip-requires
@@ -1,7 +1,7 @@
 # All dependencies needed to run rucio should be defined here
 
-SQLAlchemy==1.3.7                                 # DB backend
-alembic==1.4.1                                    # Lightweight database migration tool for SQLAlchemy
+SQLAlchemy==1.3.20                                # DB backend
+alembic==1.4.3                                    # Lightweight database migration tool for SQLAlchemy
 web.py==0.39; python_version <= '2.7'             # Python web framework for Python2
 web.py==0.40; python_version > '2.7'              # Python web framework for Python3
 python-memcached==1.59; python_version <= '2.7'   # Quick and small memcached client for Python2

diff --git a/lib/rucio/api/account.py b/lib/rucio/api/account.py
@@ -1,22 +1,28 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# -*- coding: utf-8 -*-
+# Copyright 2012-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# Authors:
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2011-2013
-# - Angelos Molfetas, <angelos.molfetas@cern.ch>, 2011
-# - Thomas Beermann, <thomas.beermann@cern.ch>, 2012
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2012
-# - Martin Barisits, <martin.barisits@cern.ch>, 2014
-# - Joaquin Bogado, <joaquin.bogado@cern.ch>, 2015
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2015-2019
-# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018
-# - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
-# - Patrick Austin, <patrick.austin@stfc.ac.uk>, 2020
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
-# PY3K COMPATIBLE
+# Authors:
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2015
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2012
+# - Martin Barisits <martin.barisits@cern.ch>, 2014
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2015
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2015-2019
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
+# - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 
 import rucio.api.permission
 import rucio.common.exception
@@ -51,7 +57,7 @@ def add_account(account, type, email, issuer, vo='def'):
 
     account = InternalAccount(account, vo=vo)
 
-    account_core.add_account(account, AccountType.from_sym(type), email)
+    account_core.add_account(account, AccountType[type.upper()], email)
 
 
 def del_account(account, issuer, vo='def'):

diff --git a/lib/rucio/api/did.py b/lib/rucio/api/did.py
@@ -17,7 +17,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2017
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2015
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2020
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013
 # - Martin Barisits <martin.barisits@cern.ch>, 2014-2020
@@ -155,9 +155,9 @@ def add_did(scope, name, type, issuer, account=None, statuses={}, meta={}, rules
                 raise rucio.common.exception.InvalidObject("Provided metadata %s doesn't match the naming convention: %s != %s" % (k, meta[k], extra_meta[k]))
 
         # Validate metadata
-        meta_core.validate_meta(meta=meta, did_type=DIDType.from_sym(type))
+        meta_core.validate_meta(meta=meta, did_type=DIDType[type.upper()])
 
-    return did.add_did(scope=scope, name=name, type=DIDType.from_sym(type), account=account or issuer,
+    return did.add_did(scope=scope, name=name, type=DIDType[type.upper()], account=account or issuer,
                        statuses=statuses, meta=meta, rules=rules, lifetime=lifetime,
                        dids=dids, rse_id=rse_id)
 
@@ -295,7 +295,7 @@ def list_new_dids(type=None, thread=None, total_threads=None, chunk_size=1000, v
     :param chunk_size: Number of requests to return per yield.
     :param vo: The VO to act on.
     """
-    dids = did.list_new_dids(did_type=type and DIDType.from_sym(type), thread=thread, total_threads=total_threads, chunk_size=chunk_size)
+    dids = did.list_new_dids(did_type=type and DIDType[type.upper()], thread=thread, total_threads=total_threads, chunk_size=chunk_size)
     for d in dids:
         if d['scope'].vo == vo:
             yield api_update_return_dict(d)

diff --git a/lib/rucio/api/identity.py b/lib/rucio/api/identity.py
@@ -1,25 +1,32 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# -*- coding: utf-8 -*-
+# Copyright 2012-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # Authors:
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2012
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2012, 2017
-# - Tomas Kouba, <tomas.kouba@cern.ch>, 2014
-# - Thomas Beermann, <thomas.beermann@cern.ch>, 2014
-# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2019
-# - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2015
+# - Tomáš Kouba <tomas.kouba@cern.ch>, 2014
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2014
+# - Martin Barisits <martin.barisits@cern.ch>, 2017
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Ruturaj Gujar <ruturaj.gujar23@gmail.com>, 2019
-#
-# PY3K COMPATIBLE
 
 """
 Interface for identity abstraction layer
 """
 
-
 from rucio.api import permission
 from rucio.common import exception
 from rucio.common.types import InternalAccount
@@ -36,7 +43,7 @@ def add_identity(identity_key, id_type, email, password=None):
     :param email: The Email address associated with the identity.
     :param password: If type==userpass, this sets the password.
     """
-    return identity.add_identity(identity_key, IdentityType.from_sym(id_type), email, password=password)
+    return identity.add_identity(identity_key, IdentityType[id_type.upper()], email, password=password)
 
 
 def del_identity(identity_key, id_type, issuer, vo='def'):
@@ -47,7 +54,7 @@ def del_identity(identity_key, id_type, issuer, vo='def'):
     :param issuer: The issuer account.
     :param vo: the VO of the issuer.
     """
-    id_type = IdentityType.from_sym(id_type)
+    id_type = IdentityType[id_type.upper()]
     kwargs = {'accounts': identity.list_accounts_for_identity(identity_key, id_type)}
     if not permission.has_permission(issuer=issuer, vo=vo, action='del_identity', kwargs=kwargs):
         raise exception.AccessDenied('Account %s can not delete identity' % (issuer))
@@ -74,7 +81,7 @@ def add_account_identity(identity_key, id_type, account, email, issuer, default=
 
     account = InternalAccount(account, vo=vo)
 
-    return identity.add_account_identity(identity=identity_key, type=IdentityType.from_sym(id_type), default=default, email=email, account=account, password=password)
+    return identity.add_account_identity(identity=identity_key, type=IdentityType[id_type.upper()], default=default, email=email, account=account, password=password)
 
 
 def del_account_identity(identity_key, id_type, account, issuer, vo='def'):
@@ -93,7 +100,7 @@ def del_account_identity(identity_key, id_type, account, issuer, vo='def'):
 
     account = InternalAccount(account, vo=vo)
 
-    return identity.del_account_identity(identity_key, IdentityType.from_sym(id_type), account)
+    return identity.del_account_identity(identity_key, IdentityType[id_type.upper()], account)
 
 
 def list_identities(**kwargs):
@@ -112,7 +119,7 @@ def get_default_account(identity_key, id_type):
     :param identity_key: The identity key name. For example x509 DN, or a username.
     :param id_type: The type of the authentication (x509, gss, userpass, ssh, saml).
     """
-    account = identity.get_default_account(identity_key, IdentityType.from_sym(id_type))
+    account = identity.get_default_account(identity_key, IdentityType[id_type.upper()])
     return account.external
 
 
@@ -125,5 +132,5 @@ def list_accounts_for_identity(identity_key, id_type):
 
     returns: A list of all accounts for the identity.
     """
-    accounts = identity.list_accounts_for_identity(identity_key, IdentityType.from_sym(id_type))
+    accounts = identity.list_accounts_for_identity(identity_key, IdentityType[id_type.upper()])
     return [account.external for account in accounts]
diff --git a/lib/rucio/api/vo.py b/lib/rucio/api/vo.py
@@ -1,4 +1,5 @@
-# Copyright 2019 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2019-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,7 @@
 # Authors:
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
-
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 
 from rucio.api.permission import has_permission
 from rucio.common import exception
@@ -79,7 +80,7 @@ def recover_vo_root_identity(root_vo, identity_key, id_type, email, issuer, defa
 
     account = InternalAccount('root', vo=root_vo)
 
-    return identity.add_account_identity(identity=identity_key, type=IdentityType.from_sym(id_type), default=default, email=email, account=account, password=password)
+    return identity.add_account_identity(identity=identity_key, type=IdentityType[id_type.upper()], default=default, email=email, account=account, password=password)
 
 
 def update_vo(updated_vo, parameters, issuer, vo='def'):

diff --git a/lib/rucio/common/utils.py b/lib/rucio/common/utils.py
@@ -53,6 +53,10 @@
 import threading
 import time
 import zlib
+
+from enum import Enum
+from logging import getLogger, Formatter
+from logging.handlers import RotatingFileHandler
 from uuid import uuid4 as uuid
 from xml.etree import ElementTree
 from logging.handlers import RotatingFileHandler
@@ -92,12 +96,6 @@
 # Extra modules: Only imported if available
 EXTRA_MODULES = {'paramiko': False}
 
-try:
-    from rucio.db.sqla.enum import EnumSymbol
-    EXTRA_MODULES['rucio.db.sqla.enum'] = True
-except ImportError:
-    EXTRA_MODULES['rucio.db.sqla.enum'] = False
-
 for extra_module in EXTRA_MODULES:
     try:
         imp.find_module(extra_module)
@@ -406,8 +404,8 @@ def default(self, obj):  # pylint: disable=E0202
             return obj.isoformat()
         elif isinstance(obj, datetime.timedelta):
             return obj.days * 24 * 60 * 60 + obj.seconds
-        elif isinstance(obj, EnumSymbol):
-            return obj.description
+        elif isinstance(obj, Enum):
+            return obj.name
         elif isinstance(obj, (InternalAccount, InternalScope)):
             return obj.external
         return json.JSONEncoder.default(self, obj)

diff --git a/lib/rucio/core/account.py b/lib/rucio/core/account.py
@@ -1,4 +1,5 @@
-# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2012-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,20 +16,20 @@
 # Authors:
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2012
 # - Angelos Molfetas <Angelos.Molfetas@cern.ch>, 2012
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2019
-# - Vincent Garonne <vgaronne@gmail.com>, 2012-2015
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2020
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2012-2015
 # - Martin Barisits <martin.barisits@cern.ch>, 2014
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2014-2019
-# - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2015
+# - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2015
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
-
 from datetime import datetime
+from enum import Enum
 from re import match
 from traceback import format_exc
 
@@ -44,7 +45,6 @@
 from rucio.core.vo import vo_exists
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import AccountStatus, AccountType
-from rucio.db.sqla.enum import EnumSymbol
 from rucio.db.sqla.session import read_session, transactional_session, stream_session
 
 from six import string_types
@@ -143,7 +143,7 @@ def update_account(account, key, value, session=None):
         raise exception.AccountNotFound('Account with ID \'%s\' cannot be found' % account)
     if key == 'status':
         if isinstance(value, string_types):
-            value = AccountStatus.from_sym(value)
+            value = AccountStatus[value]
         if value == AccountStatus.SUSPENDED:
             query.update({'status': value, 'suspended_at': datetime.utcnow()})
         elif value == AccountStatus.ACTIVE:
@@ -166,8 +166,8 @@ def list_accounts(filter={}, session=None):
     for filter_type in filter:
         if filter_type == 'account_type':
             if isinstance(filter['account_type'], string_types):
-                query = query.filter_by(account_type=AccountType.from_sym(filter['account_type']))
-            elif isinstance(filter['account_type'], EnumSymbol):
+                query = query.filter_by(account_type=AccountType[filter['account_type']])
+            elif isinstance(filter['account_type'], Enum):
                 query = query.filter_by(account_type=filter['account_type'])
 
         elif filter_type == 'identity':

diff --git a/lib/rucio/core/did.py b/lib/rucio/core/did.py
@@ -18,7 +18,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2019
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2020
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2014-2015
@@ -39,6 +39,7 @@
 import random
 import sys
 from datetime import datetime, timedelta
+from enum import Enum
 from hashlib import md5
 from re import match
 
@@ -60,7 +61,6 @@
 from rucio.core.naming_convention import validate_name
 from rucio.db.sqla import models, filter_thread_work
 from rucio.db.sqla.constants import DIDType, DIDReEvaluation, DIDAvailability, RuleState
-from rucio.db.sqla.enum import EnumSymbol
 from rucio.db.sqla.session import read_session, transactional_session, stream_session
 
 logging.basicConfig(stream=sys.stdout,
@@ -158,7 +158,7 @@ def add_dids(dids, account, session=None):
             try:
 
                 if isinstance(did['type'], string_types):
-                    did['type'] = DIDType.from_sym(did['type'])
+                    did['type'] = DIDType[did['type']]
 
                 if did['type'] == DIDType.FILE:
                     raise exception.UnsupportedOperation("Only collection (dataset/container) can be registered." % locals())
@@ -836,8 +836,8 @@ def list_new_dids(did_type, thread=None, total_threads=None, chunk_size=1000, se
 
     if did_type:
         if isinstance(did_type, string_types):
-            query = query.filter_by(did_type=DIDType.from_sym(did_type))
-        elif isinstance(did_type, EnumSymbol):
+            query = query.filter_by(did_type=DIDType[did_type])
+        elif isinstance(did_type, Enum):
             query = query.filter_by(did_type=did_type)
 
     query = filter_thread_work(session=session, query=query, total_threads=total_threads, thread_id=thread, hash_variable='name')

diff --git a/lib/rucio/core/importer.py b/lib/rucio/core/importer.py
@@ -1,4 +1,5 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2018-2020 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +15,11 @@
 #
 # Authors:
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Andrew Lister, <andrew.lister@stfc.ac.uk>, 2019
+# - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Aristeidis Fkiaras <aristeidis.fkiaras@cern.ch>, 2019
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
-#
-# PY3K COMPATIBLE
+# - Tomas Javurek <tomas.javurek@cern.ch>, 2020
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 
 from six import string_types
 from rucio.common.exception import RSEOperationNotSupported
@@ -36,7 +37,7 @@ def import_rses(rses, rse_sync_method='edit', attr_sync_method='edit', protocol_
     for rse_name in rses:
         rse = rses[rse_name]
         if isinstance(rse.get('rse_type'), string_types):
-            rse['rse_type'] = RSEType.from_string(str(rse['rse_type']))
+            rse['rse_type'] = RSEType(rse['rse_type'])
 
         if rse_module.rse_exists(rse_name, vo=vo, include_deleted=False, session=session):
             # RSE exists and is active
@@ -161,7 +162,7 @@ def import_distances(distances, vo='def', session=None):
 @transactional_session
 def import_identities(identities, account_name, old_identities, old_identity_account, account_email, session=None):
     for identity in identities:
-        identity['type'] = IdentityType.from_sym(identity['type'])
+        identity['type'] = IdentityType[identity['type'].upper()]
 
     missing_identities = [identity for identity in identities if (identity['identity'], identity['type']) not in old_identities]
     missing_identity_account = [identity for identity in identities if (identity['identity'], identity['type'], account_name) not in old_identity_account]