Skip to content

Commit

Permalink
Merge pull request #4033 from bziemons/patch-1817-switch_autotests_to…
Browse files Browse the repository at this point in the history
…_flask_api

REST API & Testing: Add Flask REST backend tests; Fix #1026
  • Loading branch information
bari12 committed Oct 8, 2020
2 parents c1f35bd + b3d89e2 commit 27c8e94
Show file tree
Hide file tree
Showing 57 changed files with 3,069 additions and 3,621 deletions.
1 change: 1 addition & 0 deletions etc/docker/test/extra/rucio.conf
Expand Up @@ -5,6 +5,7 @@ Listen 443
WSGIRestrictEmbedded On
WSGIDaemonProcess rucio processes=4 threads=4
WSGIApplicationGroup rucio
WSGIProcessGroup rucio

<VirtualHost *:443>

Expand Down
16 changes: 16 additions & 0 deletions etc/docker/test/matrix.yml
Expand Up @@ -21,7 +21,23 @@ suites:
- postgres9
- postgres12
- sqlite
REST_BACKEND:
- webpy
- flask
deny:
REST_BACKEND: flask
RDBMS:
- mysql5
- postgres9
- postgres12
- sqlite
- id: client
RDBMS: sqlite
REST_BACKEND:
- webpy
- flask
- id: multi_vo
RDBMS: postgres12
REST_BACKEND:
- webpy
- flask
1 change: 0 additions & 1 deletion etc/pip-requires-test
@@ -1,6 +1,5 @@
# All dependencies needed to develop/test rucio should be defined here

Paste==3.0.8 # Utilities for web development in pyton (3.0.8 last Py27 compatible version)
pytest==4.6.11; python_version < '3.6' # Used for testing
pytest==6.0.1; python_version >= '3.6'
Sphinx==1.8.5; python_version < '3.5' # required to build documentation (1.8.5 last Py27 compatible version)
Expand Down
10 changes: 8 additions & 2 deletions lib/rucio/core/identity.py
Expand Up @@ -114,8 +114,14 @@ def add_account_identity(identity, type, account, email, default=False, password

try:
iaa.save(session=session)
except IntegrityError:
raise exception.Duplicate('Identity pair \'%s\',\'%s\' already exists!' % (identity, type))
except IntegrityError as error:
if match('.*IntegrityError.*ORA-00001: unique constraint.*violated.*', error.args[0]) \
or match('.*IntegrityError.*UNIQUE constraint failed.*', error.args[0]) \
or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0]) \
or match('.*UniqueViolation.*duplicate key value violates unique constraint.*', error.args[0]) \
or match('.*IntegrityError.*columns? .*not unique.*', error.args[0]):
raise exception.Duplicate('Identity pair \'%s\',\'%s\' already exists!' % (identity, type))


@read_session
Expand Down
88 changes: 44 additions & 44 deletions lib/rucio/tests/common.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright 2012-2020 CERN for the benefit of the ATLAS collaboration.
# Copyright 2012-2020 CERN
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -26,19 +26,16 @@
# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020

import contextlib
import itertools
import os
import subprocess
import tempfile
from random import choice
from string import ascii_uppercase

import pytest
from paste.fixture import TestApp
from six import PY3

from rucio.client.accountclient import AccountClient
from rucio.common import exception
from rucio.common.config import config_get, config_get_bool
from rucio.common.utils import generate_uuid as uuid

skip_rse_tests_with_accounts = pytest.mark.skipif(not os.path.exists('etc/rse-accounts.cfg'), reason='fails if no rse-accounts.cfg found')
Expand All @@ -59,9 +56,6 @@ def execute(cmd):
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out = ''
err = ''
exitcode = 0

result = process.communicate()
(out, err) = result
Expand All @@ -70,42 +64,6 @@ def execute(cmd):
return exitcode, out.decode(), err.decode()


def create_accounts(account_list, user_type):
""" Registers a set of accounts
:param account_list: the list of accounts to be added
:param user_type: the type of accounts
"""
account_client = AccountClient()
for account in account_list:
try:
account_client.add_account(account, user_type, email=None)
except exception.Duplicate:
pass # Account already exists, no need to create it


def get_auth_token(account, username, password):
""" Get's an authentication token from the server
:param account: the account authenticating
:param username:the username linked to the account
:param password: the password linked to the account
:returns: the authentication token
"""
if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
vo_header = {'X-Rucio-VO': config_get('client', 'vo', raise_exception=False, default='tst')}
else:
vo_header = {}

from rucio.web.rest.authentication import APP as auth_app
mw = []
header = {'Rucio-Account': account, 'Rucio-Username': username, 'Rucio-Password': password}
header.update(vo_header)
r1 = TestApp(auth_app.wsgifunc(*mw)).get('/userpass', headers=header, expect_errors=True)
token = str(r1.header('Rucio-Auth-Token'))
return token


def account_name_generator():
""" Generate random account name.
Expand Down Expand Up @@ -172,3 +130,45 @@ def mocked_open(filename, mode='r'):
finally:
file_like_object.seek(0)
delattr(module, 'open')


def print_response(rest_response):
print('Status:', rest_response.status)
print()
nohdrs = True
for hdr, val in rest_response.headers.items():
if nohdrs:
print('Headers:')
print('-------')
nohdrs = False
print('%s: %s' % (hdr, val))

if not nohdrs:
print()

text = rest_response.get_data(as_text=True)
print(text if text else '<no content>')


def headers(*iterables):
return list(itertools.chain(*iterables))


def loginhdr(account, username, password):
yield 'X-Rucio-Account', str(account)
yield 'X-Rucio-Username', str(username)
yield 'X-Rucio-Password', str(password)


def auth(token):
yield 'X-Rucio-Auth-Token', str(token)


def vohdr(vo):
if vo:
yield 'X-Rucio-VO', str(vo)


def hdrdict(dictionary):
for key in dictionary:
yield str(key), str(dictionary[key])
140 changes: 140 additions & 0 deletions lib/rucio/tests/conftest.py
@@ -0,0 +1,140 @@
# -*- coding: utf-8 -*-
# Copyright 2020 CERN
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Authors:
# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
import importlib
import os
import traceback

import pytest


# local imports in the fixtures to make this file loadable in e.g. client tests


@pytest.fixture(scope='session')
def vo():
from rucio.common.config import config_get_bool, config_get

if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
return config_get('client', 'vo', raise_exception=False, default='tst')
else:
return 'def'


@pytest.fixture(scope='module')
def replica_client():
from rucio.client.replicaclient import ReplicaClient

return ReplicaClient()


@pytest.fixture(scope='module')
def did_client():
from rucio.client.didclient import DIDClient

return DIDClient()


@pytest.fixture
def rest_client():
from rucio.tests.common import print_response

backend = os.environ.get('REST_BACKEND', 'webpy')
if backend == 'flask':
from flask.testing import FlaskClient
from rucio.web.rest.flaskapi.v1.main import application

class WrappedFlaskClient(FlaskClient):
def __init__(self, *args, **kwargs):
super(WrappedFlaskClient, self).__init__(*args, **kwargs)

def open(self, *args, **kwargs):
response = super(WrappedFlaskClient, self).open(*args, **kwargs)
try:
print_response(response)
except Exception:
traceback.print_exc()
return response

_testing = application.testing
application.testing = True
application.test_client_class = WrappedFlaskClient
with application.test_client() as client:
yield client
application.test_client_class = None
application.testing = _testing
elif backend == 'webpy':
from werkzeug.test import Client as TestClient
from werkzeug.wrappers import BaseResponse
from rucio.web.rest.main import application as main_application

def _path_matches_endpoint(path, endpoint):
return path.startswith(endpoint + '/') or path.startswith(endpoint + '?') or path == endpoint

class WrappedTestClient(TestClient):
special_endpoints = {
'/auth': ('rucio.web.rest.authentication', None),
'/credentials': ('rucio.web.rest.credential', None),
'/nongrid_traces': ('rucio.web.rest.nongrid_trace', None),
'/ping': ('rucio.web.rest.ping', None),
'/redirect': ('rucio.web.rest.redirect', None),
'/traces': ('rucio.web.rest.trace', None),
}

def __init__(self, *args, **kwargs):
super(WrappedTestClient, self).__init__(*args, **kwargs)

def _endpoint_specials(self, path):
for endpoint_path in self.special_endpoints:
if _path_matches_endpoint(path, endpoint_path):
endpoint_module, endpoint_client = self.special_endpoints[endpoint_path]
if endpoint_client is None:
module = importlib.import_module(endpoint_module)
endpoint_client = TestClient(getattr(module, 'application'), BaseResponse)
self.special_endpoints[endpoint_path] = (module, endpoint_client)

path = path[len(endpoint_path):]
if not path.startswith('/'):
path = '/' + path

return path, endpoint_client.open
return path, super(WrappedTestClient, self).open

def open(self, *args, **kwargs):
path, open_method = self._endpoint_specials(args[0])

response = open_method(path, *args[1:], **kwargs)
try:
print_response(response)
except Exception:
traceback.print_exc()
return response

yield WrappedTestClient(main_application, BaseResponse)
else:
raise RuntimeError('Unknown rest backend ' + backend)


@pytest.fixture
def auth_token(rest_client, vo):
from rucio.tests.common import vohdr, headers, loginhdr

auth_response = rest_client.get('/auth/userpass', headers=headers(loginhdr('root', 'ddmlab', 'secret'), vohdr(vo)))
assert auth_response.status_code == 200
token = auth_response.headers.get('X-Rucio-Auth-Token')
assert token
return str(token)

0 comments on commit 27c8e94

Please sign in to comment.