Merge pull request #1033 from vingar/patch-1009-Add_python2_7_conditi…

…onal_requirements_to_core_dependencies_to_support_pip_py3_installation Release management: More work on travis (Fix issues #1009 #1017 #1017 #1018 #1019 #1020 #1012 #1022 #1024 #1025)
rucio · Apr 11, 2018 · cebd628 · cebd628
2 parents b52ab96 + d866dd0
commit cebd628
Show file tree

Hide file tree

Showing 16 changed files with 298 additions and 132 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,33 +1,115 @@
+# Copyright 2017-2018 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2017-2018
+# - Vincent Garonne <vgaronne@gmail.com>, 2017-2018
+
 sudo: required
 
-language: bash
+os:
+  - linux
+
+language: python
+
+python:
+  - 2.6
+  - 2.7
+#  - 3.4
+  - 3.5
+  - 3.6
 
 services:
   - docker
 
 install:
-  - cp etc/docker/travis/Dockerfile Dockerfile
   - git diff --name-status HEAD $(git merge-base HEAD master) | grep \.py$ | grep -v '^A' | grep -v 'conf.py' | cut -f 2 | paste -sd " " - > changed_files.txt
-  - docker build -t rucio/rucio .
+  - python --version
+  - pip --version
+  - tools/travis/install_script.sh
 
 before_script:
-  - docker run -d -p 8080:8080 -p 1521:1521 --name=oracle -e processes=1000 -e sessions=1105 -e transactions=1215 -e ORACLE_ALLOW_REMOTE=true sath89/oracle-xe-11g
-  - docker run --name=mysql -e MYSQL_ROOT_PASSWORD=secret -e MYSQL_ROOT_HOST=% -d mysql/mysql-server:5.7
-  - docker run --name=postgres -e POSTGRES_PASSWORD=secret -d postgres
-  - docker run --name=activemq -d webcenter/activemq:latest
-  - sleep 150
-  - docker run -d --link oracle:oracle --link mysql:mysql --link postgres:postgres --link activemq:activemq --name=rucio rucio/rucio
+  - tools/travis/before_script.sh
   - docker ps -a
 
 script:
-  - docker exec -it rucio /bin/sh -c "tools/travis_scripts.sh"
-
-after_script:
-  - docker stop rucio
-  - docker stop oracle
-  - docker stop mysql
-  - docker stop postgres
-  - docker stop activemq
+  - if [[ $SUITE == "syntax" ]]; then ./tools/travis/check_syntax.sh  ; fi
+  - if [[ $SUITE == "client" ]]; then nosetests -v lib/rucio/tests/test_clients.py ; fi
+  - if [[ $SUITE == "all" ]]; then docker exec -it rucio /bin/sh -c "/opt/rucio/tools/run_tests_docker.sh -1q" ; fi
+
+env:
+    - SUITE=all RDBMS=oracle
+    - SUITE=all RDBMS=mysql
+    - SUITE=all RDBMS=postgres
+    - SUITE=all RDBMS=sqlite
+    - SUITE=client RDBMS=sqlite RUCIO_HOME=`pwd`
+    - SUITE=syntax
+
+matrix:
+  fast_finish: true
+  exclude:
+  - python: 2.6
+    env: SUITE=all RDBMS=oracle
+  - python: 2.6
+    env: SUITE=all RDBMS=mysql
+  - python: 2.6
+    env: SUITE=all RDBMS=postgres
+  - python: 2.6
+    env: SUITE=all RDBMS=sqlite
+  - python: 2.6
+    env: SUITE=syntax
+  - python: 3.4
+    env:  SUITE=all RDBMS=oracle
+  - python: 3.4
+    env:  SUITE=all RDBMS=mysql
+  - python: 3.4
+    env:  SUITE=all RDBMS=postgres
+  - python: 3.4
+    env:  SUITE=all RDBMS=sqlite
+  - python: 3.4
+    env: SUITE=syntax
+  - python: 3.5
+    env: SUITE=syntax
+  - python: 3.5
+    env:  SUITE=all RDBMS=oracle
+  - python: 3.5
+    env:  SUITE=all RDBMS=mysql
+  - python: 3.5
+    env:  SUITE=all RDBMS=postgres
+  - python: 3.5
+    env:  SUITE=all RDBMS=sqlite
+  - python: 3.6
+    env: SUITE=all
+  - python: 3.5
+    env:  SUITE=all RDBMS=oracle
+  - python: 3.5
+    env:  SUITE=all RDBMS=mysql
+  - python: 3.5
+    env:  SUITE=all RDBMS=postgres
+  - python: 3.5
+    env:  SUITE=all RDBMS=sqlite
+  - python: 3.6
+    env:  SUITE=all RDBMS=oracle
+  - python: 3.6
+    env:  SUITE=all RDBMS=mysql
+  - python: 3.6
+    env:  SUITE=all RDBMS=postgres
+  - python: 3.6
+    env:  SUITE=all RDBMS=sqlite
+  allow_failures:
+  - python: 3.6
+    env: SUITE=syntax
 
 notifications:
   slack:

diff --git a/etc/docker/travis/Dockerfile b/etc/docker/travis/Dockerfile
@@ -25,66 +25,50 @@ RUN yum clean all
 RUN yum install -y epel-release.noarch
 
 RUN yum install -y gcc
-
 RUN yum install -y httpd
 RUN yum install -y python-pip gmp-devel krb5-devel httpd mod_ssl mod_auth_kerb mod_wsgi git python-devel.x86_64 openssl-devel.x86_64 gridsite which MySQL-python libaio
+RUN yum install -y python34 python34-pip python34-devel
 
 WORKDIR /opt
 
 RUN mkdir /opt/rucio
 
-COPY .pep8 /opt/rucio/
-COPY .flake8 /opt/rucio/
-COPY pylintrc /opt/rucio/
+COPY . /opt/rucio/
 COPY changed_files.txt /opt/rucio/
-COPY setup*.py /opt/rucio/
-
-RUN mkdir /opt/rucio/bin
-COPY bin /opt/rucio/bin
-
-RUN mkdir /opt/rucio/etc
-COPY etc /opt/rucio/etc
-
-RUN mkdir -p /opt/rucio/lib/rucio
-COPY lib/rucio /opt/rucio/lib/rucio
-
-RUN mkdir /opt/rucio/tools
-COPY tools /opt/rucio/tools
 
 WORKDIR /opt/rucio
 
 RUN echo "/usr/lib/oracle/12.2/client64/lib" >/etc/ld.so.conf.d/oracle.conf
 RUN ldconfig
 
 RUN pip install --upgrade pip
-RUN pip install -r tools/pip-requires-client
-RUN pip install -r tools/pip-requires
-RUN pip install -r tools/pip-requires-test
-RUN pip install cx_Oracle
-RUN pip install psycopg2
 
-RUN ln -s /opt/rucio/lib/rucio /usr/lib/python2.7/site-packages/rucio
+# Get the latest setuptools version
+# to fix the setup.py error:
+# install fails with: `install_requires` must be a string or list of strings
+RUN pip install --upgrade setuptools
+
+# Install Rucio + dependencies
+RUN pip install .[oracle,postgresql,mysql,kerberos,dev]
 
 RUN cp etc/alembic.ini.template etc/alembic.ini
-ADD etc/docker/travis/aliases-py27.conf etc/web/
+RUN cp etc/docker/travis/aliases-py27.conf etc/web/aliases-py27.conf
 
 RUN mkdir /var/log/rucio
 RUN mkdir /var/log/rucio/trace
 RUN chmod 777 /var/log/rucio
 
-ADD etc/docker/travis/httpd.conf /etc/httpd/conf/httpd.conf
-ADD etc/docker/travis/rucio.conf /etc/httpd/conf.d/rucio.conf
+RUN cp etc/docker/travis/httpd.conf /etc/httpd/conf/httpd.conf
+RUN cp etc/docker/travis/rucio.conf /etc/httpd/conf.d/rucio.conf
 
-ADD etc/docker/travis/certs/ca.pem /opt/rucio/etc/web/CERN-bundle.pem
-ADD etc/docker/travis/certs/ca.pem /opt/rucio/etc/web/ca.crt
-ADD etc/docker/travis/certs/usercert.pem /opt/rucio/etc/web/usercert.pem
+RUN cp etc/docker/travis/certs/ca.pem /opt/rucio/etc/web/CERN-bundle.pem
+RUN cp etc/docker/travis/certs/ca.pem /opt/rucio/etc/web/ca.crt
+RUN cp etc/docker/travis/certs/usercert.pem /opt/rucio/etc/web/usercert.pem
 
-ADD etc/docker/travis/certs/server.crt /etc/grid-security/hostcert.pem
-ADD etc/docker/travis/certs/server.key /etc/grid-security/hostkey.pem
+RUN cp etc/docker/travis/certs/server.crt /etc/grid-security/hostcert.pem
+RUN cp etc/docker/travis/certs/server.key /etc/grid-security/hostkey.pem
 RUN chmod 400 /etc/grid-security/hostkey.pem
 
 RUN rm /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/autoindex.conf /etc/httpd/conf.d/userdir.conf /etc/httpd/conf.d/welcome.conf
 
-ENV PATH $PATH:/opt/rucio/bin
-
 CMD ["/usr/sbin/init"]
diff --git a/etc/docker/travis/rucio_client.cfg b/etc/docker/travis/rucio_client.cfg
@@ -0,0 +1,28 @@
+[common]
+logdir = /var/log/rucio
+loglevel = DEBUG
+mailtemplatedir=etc/mail_templates
+
+[client]
+rucio_host = https://localhost:443
+auth_host = https://localhost:443
+auth_type = userpass
+username = ddmlab
+password = secret
+ca_cert = etc/web/ca.crt
+client_cert = etc/web/client.crt
+client_key = etc/web/client.key
+client_x509_proxy = $X509_USER_PROXY
+account = root
+request_retries = 3
+
+[test]
+cacert = etc/docker/travis/certs/ca.pem
+usercert = etc/docker/travis/certs/usercert.pem
+
+[policy]
+permission = generic
+schema = generic
+lfn2pfn_algorithm_default = hash
+support = https://github.com/rucio/rucio/issues
+support_rucio = https://github.com/rucio/rucio/issues
diff --git a/etc/docker/travis/rucio_mysql.cfg b/etc/docker/travis/rucio_mysql.cfg
diff --git a/etc/docker/travis/rucio_oracle.cfg b/etc/docker/travis/rucio_oracle.cfg
diff --git a/etc/docker/travis/rucio_postgres.cfg b/etc/docker/travis/rucio_postgres.cfg
diff --git a/etc/docker/travis/rucio_sqlite.cfg b/etc/docker/travis/rucio_sqlite.cfg
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
@@ -30,13 +30,11 @@
 import nose.tools
 import re
 
-from rucio import version
+from rucio.client.accountlimitclient import AccountLimitClient
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
 from rucio.common.config import config_get
 from rucio.common.utils import generate_uuid, md5
-from rucio.core.account_limit import set_account_limit
-from rucio.core.rse import get_rse_id
 from rucio.tests.common import execute, account_name_generator, rse_name_generator, file_generator, scope_name_generator
 from rucio.rse import rsemanager as rsemgr
 
@@ -56,14 +54,14 @@ def setup(self):
         self.def_rse = 'MOCK4'
         self.did_client = DIDClient()
         self.replica_client = ReplicaClient()
-
-        set_account_limit('root', get_rse_id(self.def_rse), -1)
+        self.account_client = AccountLimitClient()
+        self.account_client.set_account_limit('root', self.def_rse, -1)
 
     def test_rucio_version(self):
         """CLIENT(USER): Rucio version"""
         cmd = 'bin/rucio --version'
         exitcode, out, err = execute(cmd)
-        nose.tools.assert_equal(err, 'rucio %s\n' % version.version_string())
+        nose.tools.assert_true('rucio' in err)
 
     def test_rucio_ping(self):
         """CLIENT(USER): Rucio ping"""
@@ -516,7 +514,7 @@ def test_create_rule(self):
         exitcode, out, err = execute(cmd)
         print(out)
         # add quota
-        set_account_limit('root', get_rse_id(tmp_rse), -1)
+        self.account_client.set_account_limit('root', tmp_rse, -1)
         # add rse atributes
         cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
         print(self.marker + cmd)
@@ -529,7 +527,7 @@ def test_create_rule(self):
         exitcode, out, err = execute(cmd)
         print(out, err)
         # add quota
-        set_account_limit('root', get_rse_id(tmp_rse), -1)
+        self.account_client.set_account_limit('root', tmp_rse, -1)
         # add rse atributes
         cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
         print(self.marker + cmd)
@@ -542,7 +540,7 @@ def test_create_rule(self):
         exitcode, out, err = execute(cmd)
         print(out, err)
         # add quota
-        set_account_limit('root', get_rse_id(tmp_rse), -1)
+        self.account_client.set_account_limit('root', tmp_rse, -1)
         # add rse atributes
         cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)
         print(self.marker + cmd)
@@ -563,7 +561,7 @@ def test_create_rule(self):
 
     def test_delete_rule(self):
         """CLIENT(USER): rule deletion"""
-        set_account_limit('root', get_rse_id(self.def_rse), -1)
+        self.account_client.set_account_limit('root', self.def_rse, -1)
         tmp_file1 = file_generator()
         # add files
         cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(self.def_rse, self.user, tmp_file1)
@@ -576,8 +574,7 @@ def test_delete_rule(self):
         print(self.marker + cmd)
         exitcode, out, err = execute(cmd)
         print(out)
-
-        set_account_limit('root', get_rse_id(tmp_rse), -1)
+        self.account_client.set_account_limit('root', tmp_rse, -1)
 
         # add rse atributes
         cmd = 'rucio-admin rse set-attribute --rse {0} --key spacetoken --value ATLASSCRATCHDISK'.format(tmp_rse)

diff --git a/setup.py b/setup.py
@@ -168,11 +168,13 @@ def write_requirements():
 postgresql_extras = ['psycopg2>=2.4.2']
 mysql_extras = ['PyMySQL']
 kerberos_extras = ['kerberos>=1.2.5', 'pykerberos>=1.1.14', 'requests-kerberos>=0.11.0']
+dev_extras = parse_requirements(requirements_files=['tools/pip-requires-test', ])
 requires = parse_requirements(requirements_files=requirements_files)
 extras_require = dict(oracle=oracle_extras,
                       postgresql=postgresql_extras,
                       mysql=mysql_extras,
-                      kerberos=kerberos_extras)
+                      kerberos=kerberos_extras,
+                      dev=dev_extras)
 depend_links = parse_dependency_links(requirements_files=requirements_files)
 
 
@@ -187,7 +189,7 @@ def __init__(self, *args, **kwargs):
         self.packaging = "default value for this option"
 
     def get_file_list(self):
-        print "Chosen packaging option: " + name
+        print("Chosen packaging option: " + name)
         self.distribution.data_files = data_files
         _sdist.get_file_list(self)
 

diff --git a/tools/pip-requires b/tools/pip-requires
@@ -5,7 +5,7 @@ alembic==0.9.8              # Lightweight database migration tool for SQLAlchemy
 Mako==1.0.7                 # Templating language - Dependency of alembic
 python-editor==1.0.3        # Programmatically open an editor, capture the result
 flup==1.0.3                 # Needed to deploy web.py in lighthttpd
-web.py==0.38                # Python web framework
+web.py==0.38; python_version == '2.7' # Python web framework
 python-memcached==1.59      # Quick and small memcached client for Python
 jsonschema==2.6.0           # JSON Schema
 python-dateutil==2.6.1      # Extensions to the standard datetime module
@@ -14,7 +14,7 @@ pycrypto==2.6.1             # Cryptographic modules
 s3cmd==2.0.1                # Package built from http://s3tools.org/download
 stomp.py==4.1.20            # ActiveMQ Messaging Protocol
 dnspython==1.15.0           # To resolve ActiveMQ broker alias
-pystatsd==0.1.10            # Needed to log into graphite with more than 1 Hz
+pystatsd==0.1.10; python_version == '2.7' # Needed to log into graphite with more than 1 Hz
 pygeoip==0.3.2              # GeoIP API
 geoip2==2.7.0               # GeoIP2 API (for IPv6 support)
 ipaddress==1.0.19           # IPv4/IPv6 manipulation library - Dependency of geoip2
@@ -34,9 +34,8 @@ enum34==1.1.6               # Python 3.4 Enum backported - Dependency of grpcio
 pyasn1==0.4.2               # Dependency of oauth2client
 pyasn1-modules==0.2.1       # A collection of ASN.1-based protocols modules - Dependency of oauth2client
 rsa==3.4.2                  # Pure-Python RSA implementation
-setuptools==38.5.1          # Easily download, build, install, upgrade, and uninstall Python packages
 retrying==1.3.3             # general-purpose retrying library to simplify the task of adding retry behavior to just about anything
-functools32==3.2.3.post2    # explicitly needed on CC7
+functools32==3.2.3.post2; python_version == '2.7'    # explicitly needed on CC7
 redis==2.10.6               # Python client for Redis key-value store
 numpy==1.14.1               # Numpy for forecasting T3C
 kerberos>=1.2.5             # Kerberos high-level interface