From cc4395ed5eedc3d0385178563bbb2e7e33bef153 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Thu, 8 Aug 2019 08:17:21 +0300 Subject: [PATCH 01/19] misses for deploy instructions --- docs/source/deploy.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/deploy.rst b/docs/source/deploy.rst index ee62a3681..b2a8e51e5 100644 --- a/docs/source/deploy.rst +++ b/docs/source/deploy.rst @@ -15,7 +15,11 @@ Using vanilla docker in order to build the image - the tag can be customised: $ git clone https://github.com/CSCfi/swift-browser-ui/ $ docker build -t cscfi/swift-ui . - $ docker run -p 5050:5050 cscfi/swift-ui + $ docker run -p 8080:8080 cscfi/swift-ui + $ # or with environment variables + $ docker run -p 8080:8080 \ + -e BROWSER_START_AUTH_ENDPOINT_URL=https://pouta.csc.fi:5001/v3 \ + cscfi/swift-ui From 745554be8536da6c1ff65a9157a150d2ba643378 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Thu, 8 Aug 2019 08:18:20 +0300 Subject: [PATCH 02/19] add microbadge for docker readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a07e3f505..09b51fced 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![Build Status](https://travis-ci.com/CSCfi/swift-browser-ui.svg?branch=master)](https://travis-ci.com/CSCfi/swift-browser-ui) [![Coverage Status](https://coveralls.io/repos/github/CSCfi/swift-browser-ui/badge.svg?branch=master)](https://coveralls.io/github/CSCfi/swift-browser-ui?branch=master) [![Documentation Status](https://readthedocs.org/projects/swift-browser-ui/badge/?version=latest)](https://swift-browser-ui.readthedocs.io/en/latest/?badge=latest) +[![](https://images.microbadger.com/badges/image/cscfi/swift-ui.svg)](https://microbadger.com/images/cscfi/swift-ui "Get your own image badge on microbadger.com") + ### Description @@ -16,7 +18,8 @@ Project documentation is hosted on readthedocs: https://swift-browser-ui.rtfd.io Python 3.6+ required The dependencies mentioned in `requirements.txt` and an account that has access -rights to CSC Pouta platform, and is taking part to at least one project as +rights to [CSC Pouta](https://research.csc.fi/pouta-user-guide) platform, +and is taking part to at least one project as object stoarge is project specific. ### Usage From f42fec2a8ab44303b6b3ae74d2a0c3a8d2f127b9 Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 7 Aug 2019 14:16:33 +0300 Subject: [PATCH 03/19] Add tests for swift_browser_ui.front. --- tests/test_front.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_front.py diff --git a/tests/test_front.py b/tests/test_front.py new file mode 100644 index 000000000..f81153760 --- /dev/null +++ b/tests/test_front.py @@ -0,0 +1,54 @@ +"""Module for testing ``swift_browser_ui.front``.""" + + +import unittest +import os + +from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop + +from swift_browser_ui.server import servinit + + +class FrontendTestCase(AioHTTPTestCase): + """Test frontend.""" + + async def get_application(self): + """Retrieve web Application for test.""" + return await servinit() + + @staticmethod + def return_true(_): + """.""" + return True + + @unittest_run_loop + async def test_browse(self): + """Test /browse handler.""" + patch_setd = unittest.mock.patch("swift_browser_ui.front.setd", new={ + "static_directory": os.getcwd() + "/swift_browser_ui_frontend" + }) + patch_check = unittest.mock.patch( + "swift_browser_ui.front.session_check", + new=self.return_true + ) + with patch_setd, patch_check: + response = await self.client.request("GET", "/browse") + self.assertEqual(response.status, 200) + self.assertEqual(response.headers["Content-type"], + "text/html") + + @unittest_run_loop + async def test_index(self): + """Test / handler.""" + patch_setd = unittest.mock.patch("swift_browser_ui.front.setd", new={ + "static_directory": os.getcwd() + "/swift_browser_ui_frontend" + }) + patch_check = unittest.mock.patch( + "swift_browser_ui.front.session_check", + new=self.return_true + ) + with patch_setd, patch_check: + response = await self.client.request("GET", "/") + self.assertEqual(response.status, 200) + self.assertEqual(response.headers["Content-type"], + "text/html") From cfaef76e4cbcba19ee3db1e9419b30322d6e9b7c Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 7 Aug 2019 14:57:05 +0300 Subject: [PATCH 04/19] Add unit tests for CSRF protection in _convenience. --- tests/creation.py | 24 ++++++++++++++ tests/test_convenience.py | 66 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/tests/creation.py b/tests/creation.py index ed28f7e6b..2590c2fef 100644 --- a/tests/creation.py +++ b/tests/creation.py @@ -16,6 +16,30 @@ from .mockups import Mock_Request, Mock_Service, Mock_Session +def add_csrf_to_cookie(cookie, req, bad_sign=False): + """Add specified csrf test variables to cookie.""" + # Getting options as a set + cookie["referer"] = "http://localhost:8080" + if bad_sign: + cookie["signature"] = "incorrect" + else: + cookie["signature"] = (hashlib.sha256((cookie["id"] + + cookie["referer"] + + req.app["Salt"]) + .encode('utf-8')) + .hexdigest()) + return cookie + + +def encrypt_cookie(cookie, req): + """Add encrypted cookie to request.""" + cookie_crypted = \ + req.app["Crypt"].encrypt( + json.dumps(cookie).encode('utf-8') + ).decode('utf-8') + req.cookies["S3BROW_SESSION"] = cookie_crypted + + def get_request_with_fernet(): """Create a request with a working fernet object.""" ret = Mock_Request() diff --git a/tests/test_convenience.py b/tests/test_convenience.py index 7ead5206e..0e57131b1 100644 --- a/tests/test_convenience.py +++ b/tests/test_convenience.py @@ -4,7 +4,10 @@ import hashlib import os import unittest + from aiohttp.web import HTTPUnauthorized, Response +from aiohttp.web import HTTPForbidden + import cryptography.fernet from swiftclient.service import SwiftService from keystoneauth1.session import Session @@ -14,10 +17,12 @@ from swift_browser_ui._convenience import get_availability_from_token from swift_browser_ui._convenience import initiate_os_service from swift_browser_ui._convenience import initiate_os_session +from swift_browser_ui._convenience import check_csrf from swift_browser_ui.settings import setd from .creation import get_request_with_fernet from .creation import get_full_crypted_session_cookie +from .creation import add_csrf_to_cookie, encrypt_cookie from .mockups import mock_token_output, urlopen @@ -222,3 +227,64 @@ def test_initiate_os_service(self): sess_mock = unittest.mock.MagicMock(Session) ret = initiate_os_service(sess_mock()) self.assertIsInstance(ret, SwiftService) # nosec + + def test_check_csrf_os_skip(self): + """Test check_csrf when skipping referer from OS.""" + with unittest.mock.patch("swift_browser_ui._convenience.setd", new={ + "auth_endpoint_url": "http://example-auth.exampleosep.com:5001/v3" + }): + testreq = get_request_with_fernet() + cookie, _ = generate_cookie(testreq) + cookie = add_csrf_to_cookie(cookie, testreq) + encrypt_cookie(cookie, testreq) + testreq.headers["Referer"] = "http://example-auth.exampleosep.com" + self.assertTrue(check_csrf(testreq)) + + def test_check_csrf_incorrect_referer(self): + """Test check_csrf when Referer header is incorrect.""" + with unittest.mock.patch("swift_browser_ui._convenience.setd", new={ + "auth_endpoint_url": "http://example-auth.exampleosep.com:5001/v3" + }): + testreq = get_request_with_fernet() + cookie, _ = generate_cookie(testreq) + cookie = add_csrf_to_cookie(cookie, testreq) + encrypt_cookie(cookie, testreq) + testreq.headers["Referer"] = "http://notlocaclhost:8080" + with self.assertRaises(HTTPForbidden): + check_csrf(testreq) + + def test_check_csrf_incorrect_signature(self): + """Test check_csrf when signature doesn't match.""" + with unittest.mock.patch("swift_browser_ui._convenience.setd", new={ + "auth_endpoint_url": "http://example-auth.exampleosep.com:5001/v3" + }): + testreq = get_request_with_fernet() + cookie, _ = generate_cookie(testreq) + cookie = add_csrf_to_cookie(cookie, testreq, bad_sign=True) + encrypt_cookie(cookie, testreq) + testreq.headers["Referer"] = "http://localhost:8080" + with self.assertRaises(HTTPForbidden): + check_csrf(testreq) + + def test_check_csrf_no_referer(self): + """Test check_csrf when no Referer header is present.""" + with unittest.mock.patch("swift_browser_ui._convenience.setd", new={ + "auth_endpoint_url": "http://example-auth.exampleosep.com:5001/v3" + }): + testreq = get_request_with_fernet() + cookie, _ = generate_cookie(testreq) + cookie = add_csrf_to_cookie(cookie, testreq) + encrypt_cookie(cookie, testreq) + self.assertTrue(check_csrf(testreq)) + + def test_check_csrf_correct_referer(self): + """Test check_csrf when the session is valid.""" + with unittest.mock.patch("swift_browser_ui._convenience.setd", new={ + "auth_endpoint_url": "http://example-auth.exampleosep.com:5001/v3" + }): + testreq = get_request_with_fernet() + cookie, _ = generate_cookie(testreq) + cookie = add_csrf_to_cookie(cookie, testreq) + encrypt_cookie(cookie, testreq) + testreq.headers["Referer"] = "http://localhost:8080" + self.assertTrue(check_csrf(testreq)) From 01d7ed99778d1e371d87e2c34f9d97c70d01d7d5 Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 7 Aug 2019 14:59:06 +0300 Subject: [PATCH 05/19] Remove return value check for check_csrf in api_check. The return value check is not necessary, since all the failure conditions present in check_csrf lead to an exception, there is no viable execution path which could lead to the check being not True while having the check_csrf function not throw. --- swift_browser_ui/_convenience.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/swift_browser_ui/_convenience.py b/swift_browser_ui/_convenience.py index be34357f3..47c50f93d 100644 --- a/swift_browser_ui/_convenience.py +++ b/swift_browser_ui/_convenience.py @@ -140,12 +140,7 @@ def api_check(request): try: if decrypt_cookie(request)["id"] in request.app['Sessions']: session = decrypt_cookie(request)["id"] - if not check_csrf(request): - raise aiohttp.web.HTTPUnauthorized( - headers={ - "WWW-Authenticate": 'Bearer realm="/", charset="UTF-8"' - } - ) + check_csrf(request) ret = session if 'ST_conn' not in request.app['Creds'][session].keys(): raise aiohttp.web.HTTPUnauthorized( From 6095936290e01c66a191b0cb73b049234a4e972f Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 7 Aug 2019 15:07:38 +0300 Subject: [PATCH 06/19] Add missing tests for missed api_check lines. Api session validity check function unit tests were not properly formed, leading to lower than expected testing coverage. The edge cases of the session validity checks are now tested for. --- tests/test_convenience.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_convenience.py b/tests/test_convenience.py index 0e57131b1..ce3820512 100644 --- a/tests/test_convenience.py +++ b/tests/test_convenience.py @@ -126,10 +126,16 @@ def test_api_check_raise_on_invalid_cookie(self): with self.assertRaises(HTTPUnauthorized): api_check(testreq) - # NOTE: The order of operations for these tests is significant - # (i.e. there is a reason some of the placeholders are missing - # in the test functions, - # to enable testing correct raising order in the same time) + def test_api_check_raise_on_invalid_fernet(self): + """Test raise if the cryptographic key has changed.""" + testreq = get_request_with_fernet() + _, testreq.cookies['S3BROW_SESSION'] = generate_cookie(testreq) + testreq.app['Crypt'] = cryptography.fernet.Fernet( + cryptography.fernet.Fernet.generate_key() + ) + with self.assertRaises(HTTPUnauthorized): + api_check(testreq) + def test_api_check_raise_on_no_connection(self): """Test raise if there's no existing OS connection on an API call.""" testreq = get_request_with_fernet() @@ -153,6 +159,7 @@ def test_api_check_raise_on_no_session(self): session = cookie["id"] testreq.app['Sessions'] = [session] testreq.app['Creds'][session] = {} + testreq.app['Creds'][session]['ST_conn'] = "placeholder" testreq.app['Creds'][session]['Avail'] = "placeholder" with self.assertRaises(HTTPUnauthorized): api_check(testreq) @@ -166,6 +173,8 @@ def test_api_check_raise_on_no_avail(self): session = cookie["id"] testreq.app['Creds'][session] = {} testreq.app['Sessions'] = [session] + testreq.app['Creds'][session]['ST_conn'] = "placeholder" + testreq.app['Creds'][session]['OS_sess'] = "placeholder" with self.assertRaises(HTTPUnauthorized): api_check(testreq) From 4c06b05a2bd6ed84b7c8fc427a6a93150f1c0f44 Mon Sep 17 00:00:00 2001 From: --global Date: Wed, 7 Aug 2019 17:15:29 +0300 Subject: [PATCH 07/19] Increase test coverage in api tests. Added tests for testing openstack active project api call, unicode null filtering and swift error handling in container listing. --- swift_browser_ui/api.py | 2 +- tests/mockups.py | 19 +++++++++++++------ tests/test_api.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/swift_browser_ui/api.py b/swift_browser_ui/api.py index 4ca14924f..32bfc2cd0 100644 --- a/swift_browser_ui/api.py +++ b/swift_browser_ui/api.py @@ -62,7 +62,7 @@ async def swift_list_buckets(request): return aiohttp.web.json_response(cont) except SwiftError: - return aiohttp.web.json_response([]) + raise aiohttp.web.HTTPNotFound() async def swift_list_objects(request): diff --git a/tests/mockups.py b/tests/mockups.py index 4b663272a..865acbbda 100644 --- a/tests/mockups.py +++ b/tests/mockups.py @@ -226,6 +226,7 @@ def init_with_data( size_range=(0, 0), container_name_prefix="test-container-", object_name_prefix=None, # None for just the hash as name + has_content_type=None, ): """Initialize the Mock_Service instance with some test data.""" for i in range(0, containers): @@ -241,14 +242,17 @@ def init_with_data( oname = object_name_prefix + ohash else: oname = ohash - to_add.append({ + to_append = { "hash": ohash, "name": oname, "last_modified": datetime.datetime.now().isoformat(), "bytes": random.randint( # nosec size_range[0], size_range[1] - ), - }) + ) + } + if has_content_type: + to_append["content_type"] = has_content_type + to_add.append(to_append) self.containers[container_name_prefix + str(i)] = to_add @@ -269,12 +273,15 @@ def list(self, container=None, options=None): ret = [] try: for i in self.containers[container]: - ret.append({ + to_append = { "hash": i["hash"], "name": i["name"], "last_modified": i["last_modified"], - "bytes": i["bytes"] - }) + "bytes": i["bytes"], + } + if "content_type" in i.keys(): + to_append["content_type"] = i["content_type"] + ret.append(to_append) return [{ "listing": ret }] diff --git a/tests/test_api.py b/tests/test_api.py index a57f4d432..45b0fcf59 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3,15 +3,19 @@ import json import hashlib import os +import unittest from aiohttp.web import HTTPNotFound import asynctest +from swiftclient.service import SwiftError + from swift_browser_ui.api import get_os_user, os_list_projects from swift_browser_ui.api import swift_list_buckets, swift_list_objects from swift_browser_ui.api import swift_download_object from swift_browser_ui.api import get_metadata from swift_browser_ui.api import get_project_metadata +from swift_browser_ui.api import get_os_active_project from swift_browser_ui.settings import setd from .creation import get_request_with_mock_openstack @@ -48,6 +52,13 @@ async def test_list_containers_correct(self): # Test if return all the correct values from the mock service self.assertEqual(buckets, comp) + async def test_list_containers_swift_error(self): + """Test function swift_list_buckets when raising SwiftError.""" + self.request.app['Creds'][self.cookie]['ST_conn'].list = \ + unittest.mock.Mock(side_effect=SwiftError("...")) + with self.assertRaises(HTTPNotFound): + _ = await swift_list_buckets(self.request) + async def test_list_objects_correct(self): """Test function swift_list_objetcs with a correct query.""" self.request.app['Creds'][self.cookie]['ST_conn'].init_with_data( @@ -67,6 +78,22 @@ async def test_list_objects_correct(self): ] self.assertEqual(objects, comp) + async def test_list_objects_with_unicode_nulls(self): + """Test function swift_list_objects with unicode nulls in type.""" + self.request.app["Creds"][self.cookie]["ST_conn"].init_with_data( + containers=1, + object_range=(0, 10), + size_range=(65535, 262144), + has_content_type="text/html", + ) + self.request.query["bucket"] = "test-container-0" + response = await swift_list_objects(self.request) + objects = json.loads(response.text) + self.assertEqual( + objects[0]["content_type"], + "text/html", + ) + async def test_list_without_containers(self): """Test function list buckets on a project without object storage.""" self.request.app['Creds'][self.cookie]['ST_conn'].init_with_data( @@ -335,3 +362,16 @@ async def test_get_project_metadata(self): resp = json.loads(resp.text) self.assertEqual(resp, comp) + + async def test_get_os_active_project(self): + """Test active project API endpoint.""" + self.request.app["Creds"][self.cookie]["active_project"] = \ + "placeholder" + resp = await get_os_active_project(self.request) + text = json.loads(resp.text) + self.assertEqual(resp.status, 200) + self.assertEqual(text, "placeholder") + + def tearDown(self): + self.cookie = None + self.request = None From 8e6d22f77e641494e3dd2fcd26059781ca23379d Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 8 Aug 2019 12:37:45 +0300 Subject: [PATCH 08/19] Add tests for token rescope. --- tests/test_api.py | 1 + tests/test_login.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 45b0fcf59..ffe33ecfe 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -373,5 +373,6 @@ async def test_get_os_active_project(self): self.assertEqual(text, "placeholder") def tearDown(self): + """Test teardown.""" self.cookie = None self.request = None diff --git a/tests/test_login.py b/tests/test_login.py index 887f36fb1..018da2021 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -3,18 +3,18 @@ import hashlib import os import unittest - +import json import asynctest -from aiohttp.web import HTTPClientError +from aiohttp.web import HTTPClientError, HTTPForbidden import swift_browser_ui.login import swift_browser_ui.settings - from .creation import get_request_with_fernet, get_request_with_mock_openstack from .mockups import return_project_avail from .mockups import return_invalid +from .mockups import mock_token_project_avail _path = "/auth/OS-FEDERATION/identity_providers/haka/protocols/saml2/websso" @@ -276,3 +276,40 @@ async def test_handle_logout(self): self.assertEqual(resp.headers['Location'], "/") sess.invalidate.assert_called_once() self.assertNotIn(cookie, req.app['Sessions']) + + async def test_token_rescope_not_available(self): + """Test the token rescope function.""" + session, req = get_request_with_mock_openstack() + req.app["Creds"][session]["Avail"] = \ + json.loads(mock_token_project_avail) + req.query["project"] = "non-existent-project" + with self.assertRaises(HTTPForbidden): + await swift_browser_ui.login.token_rescope(req) + + async def test_token_rescope_correct(self): + """Test the token rescope function with correct request.""" + session, req = get_request_with_mock_openstack() + req.app["Creds"][session]["Avail"] = \ + json.loads(mock_token_project_avail) + req.query["project"] = "wol" + req.app["Creds"][session]["Token"] = "not_actually_a_token" # nosec + + # Set up mockups + sess_mock = unittest.mock.MagicMock("keystoneauth.session.Session") + req.app["Creds"][session]["OS_sess"] = sess_mock() + patch_os_auth = unittest.mock.patch( + "swift_browser_ui.login.initiate_os_service", + new=unittest.mock.MagicMock( + swift_browser_ui._convenience.initiate_os_service + ) + ) + patch_os_sess = unittest.mock.patch( + "swift_browser_ui.login.initiate_os_session", + new=unittest.mock.MagicMock( + swift_browser_ui._convenience.initiate_os_session + ) + ) + with patch_os_auth, patch_os_sess: + resp = await swift_browser_ui.login.token_rescope(req) + self.assertEqual(resp.status, 303) + self.assertEqual(resp.headers["Location"], "/browse") From 53b5a30da1b4bd58a734381adb241a6ca0fc2143 Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 8 Aug 2019 12:51:23 +0300 Subject: [PATCH 09/19] Bump to v0.3.1 --- docs/source/conf.py | 2 +- swift_browser_ui/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9c3fbf29f..e6af9b5ae 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ author = 'CSC Developers' # The full version, including alpha/beta/rc tags -version = release = '0.3.0' +version = release = '0.3.1' # -- General configuration --------------------------------------------------- diff --git a/swift_browser_ui/__init__.py b/swift_browser_ui/__init__.py index 329bdb253..5690bce12 100644 --- a/swift_browser_ui/__init__.py +++ b/swift_browser_ui/__init__.py @@ -7,6 +7,6 @@ __name__ = 'swift_browser_ui' -__version__ = '0.3.0' +__version__ = '0.3.1' __author__ = 'CSC Developers' __license__ = 'MIT License' From 63333c9c9ad11088b01b67a8d18f75ea74bf71cb Mon Sep 17 00:00:00 2001 From: --global Date: Thu, 8 Aug 2019 13:00:16 +0300 Subject: [PATCH 10/19] Force at minimum one object in unicode null tests. --- tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index ffe33ecfe..e4888a197 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -82,7 +82,7 @@ async def test_list_objects_with_unicode_nulls(self): """Test function swift_list_objects with unicode nulls in type.""" self.request.app["Creds"][self.cookie]["ST_conn"].init_with_data( containers=1, - object_range=(0, 10), + object_range=(1, 10), size_range=(65535, 262144), has_content_type="text/html", ) From ad7aaa587d71b614284c596aad39e9ad8e113363 Mon Sep 17 00:00:00 2001 From: --global Date: Fri, 9 Aug 2019 10:15:32 +0300 Subject: [PATCH 11/19] Add a locust file for stressing the test server. --- performance_tests/locustfile.py | 89 +++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 performance_tests/locustfile.py diff --git a/performance_tests/locustfile.py b/performance_tests/locustfile.py new file mode 100644 index 000000000..30e648dd5 --- /dev/null +++ b/performance_tests/locustfile.py @@ -0,0 +1,89 @@ +"""Locust file for testing the backend API performace.""" + +from locust import HttpLocust, TaskSet + + +def login(l_instance): + """Handle locust user login.""" + l_instance.client.post( + "/login/websso", + {"token": "the_actual_token_doesn't_matter"} + ) + + +def logout(l_instance): + """Handle locust user logout.""" + l_instance.client.get( + "/login/kill" + ) + + +def api_containers(l_instance): + """Get container listing.""" + l_instance.client.get( + "/api/buckets" + ) + + +def api_objects(l_instance): + """Get object listing.""" + l_instance.client.get( + "/api/objects?bucket=test-container-0" + ) + + +def api_active(l_instance): + """Get active project.""" + l_instance.client.get( + "/api/active" + ) + + +def api_projects(l_instance): + """Get available projects.""" + l_instance.client.get( + "/api/projects" + ) + + +def api_username(l_instance): + """Get the username.""" + l_instance.client.get( + "/api/username" + ) + + +def api_project_meta(l_instance): + """Get the project metadata.""" + l_instance.client.get( + "/api/get-project-meta" + ) + + +class UserBehaviour(TaskSet): + """Locust task class for swift-browser-ui user case.""" + + tasks = { + api_containers: 1, + api_objects: 5, + api_active: 1, + api_projects: 1, + api_username: 1, + api_project_meta: 2, + } + + def on_start(self): + """Handle website login.""" + login(self) + + def on_stop(self): + """Handle website logout.""" + logout(self) + + +class APIUser(HttpLocust): + """Locust API user class.""" + + task_set = UserBehaviour + min_wait = 100 + max_wait = 1000 From 7374205b6a1fa7b05ee5b25d7b2f849217e13727 Mon Sep 17 00:00:00 2001 From: --global Date: Fri, 9 Aug 2019 13:41:59 +0300 Subject: [PATCH 12/19] Update gitignore to ignore some additional files --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index a0d6cccf6..1edf02c74 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,8 @@ venv.bak/ src/server.py +# dia temporary files +*.dia~ + +# environment variable files +setenvs* From ea84c3882be209efc3e7060c04dd9ff86966663a Mon Sep 17 00:00:00 2001 From: --global Date: Fri, 9 Aug 2019 14:30:54 +0300 Subject: [PATCH 13/19] Add unit tests for the error middleware handler. --- tests/test_middlewares.py | 96 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/test_middlewares.py diff --git a/tests/test_middlewares.py b/tests/test_middlewares.py new file mode 100644 index 000000000..84bad576b --- /dev/null +++ b/tests/test_middlewares.py @@ -0,0 +1,96 @@ +"""Module for testing ``swift_browser_ui.middlewares``.""" + + +from aiohttp.web import HTTPUnauthorized, HTTPForbidden, HTTPNotFound +from aiohttp.web import Response, HTTPClientError, FileResponse +import asynctest + +from swift_browser_ui.middlewares import error_middleware +from swift_browser_ui.front import index + + +async def return_401_handler(with_exception): + """Return an HTTP401 error.""" + if with_exception: + raise HTTPUnauthorized() + return Response( + status=401 + ) + + +async def return_403_handler(with_exception): + """Return an HTTP403 error.""" + if with_exception: + raise HTTPForbidden() + return Response( + status=403 + ) + + +async def return_404_handler(with_exception): + """Return or raise an HTTP404 error.""" + if with_exception: + raise HTTPNotFound() + return Response( + status=404 + ) + + +async def return_400_handler(with_exception): + """Return or raise an HTTP400 error.""" + if with_exception: + raise HTTPClientError() + return Response( + status=400 + ) + + +class MiddlewareTestClass(asynctest.TestCase): + """Testing the error middleware.""" + + async def test_401_return(self): + """Test 401 middleware when the 401 status is returned.""" + resp = await error_middleware(None, return_401_handler) + self.assertEqual(resp.status, 401) + self.assertIsInstance(resp, FileResponse) + + async def test_401_exception(self): + """Test 401 middleware when the 401 status is risen.""" + resp = await error_middleware(True, return_401_handler) + self.assertEqual(resp.status, 401) + self.assertIsInstance(resp, FileResponse) + + async def test_403_return(self): + """Test 403 middleware when the 403 status is returned.""" + resp = await error_middleware(None, return_403_handler) + self.assertEqual(resp.status, 403) + self.assertIsInstance(resp, FileResponse) + + async def test_403_exception(self): + """Test 403 middleware when the 403 status is risen.""" + resp = await error_middleware(True, return_403_handler) + self.assertEqual(resp.status, 403) + self.assertIsInstance(resp, FileResponse) + + async def test_404_return(self): + """Test 404 middleware when the 404 status is returned.""" + resp = await error_middleware(None, return_404_handler) + self.assertEqual(resp.status, 404) + self.assertIsInstance(resp, FileResponse) + + async def test_404_exception(self): + """Test 404 middlewrae when the 404 status is risen.""" + resp = await error_middleware(True, return_404_handler) + self.assertEqual(resp.status, 404) + self.assertIsInstance(resp, FileResponse) + + async def test_error_middleware_no_error(self): + """Test the general error middleware with correct status.""" + resp = await error_middleware(None, index) + self.assertEqual(resp.status, 200) + self.assertIsInstance(resp, FileResponse) + + async def test_error_middleware_non_handled_raise(self): + """Test the general error middleware with other status code.""" + with self.assertRaises(HTTPClientError): + await error_middleware(True, return_400_handler) From 9a7e00dc9d22986a4293b02343ccd53033c83d31 Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 12 Aug 2019 10:22:50 +0300 Subject: [PATCH 14/19] Re-enable HTTPS as an option. HTTPS was previously disabled in standalone mode (not running with a TLS termination proxy) due to difficulties in getting it properly running. Now the problems are fixed and running with HTTPS has been enabled again. Also added CLI options for specifying the SSL certificates for HTTPS. --- swift_browser_ui/server.py | 50 ++++++++++++++++++++++++-------------- swift_browser_ui/shell.py | 21 ++++++++++++++-- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/swift_browser_ui/server.py b/swift_browser_ui/server.py index b0d2f7ca9..8c436fe3b 100644 --- a/swift_browser_ui/server.py +++ b/swift_browser_ui/server.py @@ -1,13 +1,13 @@ """swift_browser_ui server related convenience functions.""" # Generic imports -# import ssl import logging import time import sys import asyncio import hashlib import os +import ssl import uvloop import cryptography.fernet @@ -124,23 +124,37 @@ async def servinit(): return app -# def run_server_secure(app): -# """ -# Run the server securely with a given ssl context. - -# While this function is incomplete, the project is safe to run in -# production only via a TLS termination proxy with e.g. NGINX. -# """ - # Setup ssl context - # sslcontext = ssl.create_default_context() - # sslcontext.set_ciphers( - # 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE' + - # '-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-' + - # 'AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-' + - # 'SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-' + - # 'RSA-AES128-SHA256' - # ) - # aiohttp.web.run_app(app, ssl_context=sslcontext) +def run_server_secure(app, cert_file, cert_key): + """ + Run the server securely with a given ssl context. + + While this function is incomplete, the project is safe to run in + production only via a TLS termination proxy with e.g. NGINX. + """ + logger = logging.getLogger("swift-browser-ui") + logger.debug("Running server securely.") + logger.debug("Setting up SSL context for the server.") + sslcontext = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) + cipher_str = ( + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE' + + '-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-' + + 'AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-' + + 'SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-' + + 'RSA-AES128-SHA256' + ) + logger.debug( + "Setting following ciphers for SSL context: \n%s", + cipher_str + ) + sslcontext.set_ciphers(cipher_str) + logger.debug("Loading cert chain.") + sslcontext.load_cert_chain(cert_file, cert_key) + aiohttp.web.run_app( + app, + access_log=aiohttp.web.logging.getLogger('aiohttp.access'), + port=setd['port'], + ssl_context=sslcontext, + ) def run_server_insecure(app): diff --git a/swift_browser_ui/shell.py b/swift_browser_ui/shell.py index 5159c3288..c06fdecc3 100644 --- a/swift_browser_ui/shell.py +++ b/swift_browser_ui/shell.py @@ -10,7 +10,7 @@ from .__init__ import __version__ from .settings import setd, set_key, FORMAT -from .server import servinit, run_server_insecure +from .server import servinit, run_server_insecure, run_server_secure from ._convenience import setup_logging as conv_setup_logging @@ -92,6 +92,18 @@ def cli(verbose, debug, logfile): @click.option( '--set-session-devmode', is_flag=True, default=False, hidden=True, ) +@click.option( + '--secure', is_flag=True, default=False, + help="Enable secure running, i.e. enable HTTPS." +) +@click.option( + '--ssl-cert-file', default=None, type=str, + help="Specify the certificate to use with SSL." +) +@click.option( + '--ssl-cert-key', default=None, type=str, + help="Specify the certificate key to use with SSL." +) def start( port, auth_endpoint_url, @@ -99,6 +111,9 @@ def start( dry_run, set_origin_address, set_session_devmode, + secure, + ssl_cert_file, + ssl_cert_key, ): """Start the browser backend and server.""" logging.debug( @@ -129,8 +144,10 @@ def start( logging.debug( "Running settings directory:%s", str(setd) ) - if not dry_run: + if not dry_run and not secure: run_server_insecure(servinit()) + if not dry_run and secure: + run_server_secure(servinit(), ssl_cert_file, ssl_cert_key) def main(): From 3434b55c0a8588ba7f52653ef3f1d908b2a74498 Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 12 Aug 2019 10:30:16 +0300 Subject: [PATCH 15/19] Update documentation after introducing SSL. --- docs/source/instructions.rst | 9 ++++----- docs/source/usage.rst | 9 +++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/source/instructions.rst b/docs/source/instructions.rst index 3c98670e0..bc64005a1 100644 --- a/docs/source/instructions.rst +++ b/docs/source/instructions.rst @@ -55,11 +55,10 @@ For the Pouta production environment for testing unsecurely without trust:: Setting up TLS termination proxy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The backend itself is not meant to be run as standalone in a production -environment. Therefore in a running config a TLS termination proxy should be -used to make the service secure. Setting up TLS termination is outside the -scope of this documentation, but a few useful links are provided along with -the necessary configs regarding this service. [#]_ [#]_ +The backend can be run in secure mode, i.e. with HTTPS enabled, but for +scaling up a TLS termination proxy is recommended. Setting up TLS termination +is outside the scope of this documentation, but a few useful links are +provided along with the necessary configs regarding this service. [#]_ [#]_ Scaling up the service ---------------------- diff --git a/docs/source/usage.rst b/docs/source/usage.rst index a36efb874..82a41bfcb 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -68,6 +68,9 @@ The following command line arguments are available for server startup. trusted_dashboards in the specified address. --set-origin-address TEXT Set the address that the program will be redirected to from WebSSO + --secure Enable secure running, i.e. enable HTTPS. + --ssl-cert-file TEXT Specify the certificate to use with SSL. + --ssl-cert-key TEXT Specify the certificate key to use with SSL. --help Show this message and exit. @@ -81,5 +84,11 @@ The following command line arguments are available for server startup. authentication endpoint, i.e. if the program has been listed on the respective Openstack keystone trusted_dashboard list. [#]_ +--secure Enable HTTPS on the server, to enable secure + requests if there's no TLS termination proxy. +--ssl-cert-file TEXT Specify SSL certificate file. Required when + running in secure mode. +--ssl-cert-key TEXT Specify SSL certificate key. Required when + running in secure mode. .. [#] https://docs.openstack.org/keystone/pike/advanced-topics/federation/websso.html From 234c6e6904bc276936e102b8aa276ce5f9ebb244 Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 12 Aug 2019 11:11:32 +0300 Subject: [PATCH 16/19] Add server run and shutdown function tests. --- tests/test_server.py | 52 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/test_server.py b/tests/test_server.py index 1de067754..c1e153f3f 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -6,13 +6,19 @@ import os +import unittest +import ssl from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop +import aiohttp import asynctest -from swift_browser_ui.server import servinit +from swift_browser_ui.server import servinit, run_server_insecure +from swift_browser_ui.server import kill_sess_on_shutdown, run_server_secure from swift_browser_ui.settings import setd +from .creation import get_request_with_mock_openstack + # Set static folder in settings so it can be tested setd['static_directory'] = os.getcwd() + '/swift_browser_ui_frontend' @@ -29,6 +35,50 @@ async def test_servinit(self): self.assertTrue(app is not None) +class TestServerShutdownHandler(asynctest.TestCase): + """Test case for the server graceful shutdown handler.""" + + async def test_kill_sess_on_shutdown(self): + """Test kill_sess_on_shutdown function.""" + session, req = get_request_with_mock_openstack() + + await kill_sess_on_shutdown(req.app) + + self.assertNotIn(session, req.app["Creds"].keys()) + + +class TestRunServerFunctions(unittest.TestCase): + """Test class for server run functions.""" + + @staticmethod + def mock_ssl_context_creation(purpose=None): + """Return a MagicMock instance of an ssl context.""" + return unittest.mock.MagicMock(ssl.create_default_context)() + + def test_run_server_secure(self): + """Test run_server_secure function.""" + run_app_mock = unittest.mock.MagicMock(aiohttp.web.run_app) + patch_run_app = unittest.mock.patch( + "swift_browser_ui.server.aiohttp.web.run_app", run_app_mock + ) + patch_ssl_defcontext = unittest.mock.patch( + "swift_browser_ui.server.ssl.create_default_context", + self.mock_ssl_context_creation + ) + with patch_run_app, patch_ssl_defcontext: + run_server_secure(None, None, None) + run_app_mock.assert_called_once() + + def test_run_server_insecure(self): + """Test run_server_insecure function.""" + run_app_mock = unittest.mock.MagicMock(aiohttp.web.run_app) + with unittest.mock.patch( + "swift_browser_ui.server.aiohttp.web.run_app", run_app_mock + ): + run_server_insecure(None) + run_app_mock.assert_called_once() + + # After testing the server initialization, we can use the correctly starting # server for testing other modules. class AppTestCase(AioHTTPTestCase): From 24c521a7ca32e077d7eacf32fa5f2e9455fc339d Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 19 Aug 2019 13:22:47 +0300 Subject: [PATCH 17/19] Add source for the SSL/TLS ciphers. --- swift_browser_ui/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/swift_browser_ui/server.py b/swift_browser_ui/server.py index 8c436fe3b..ee210534f 100644 --- a/swift_browser_ui/server.py +++ b/swift_browser_ui/server.py @@ -131,6 +131,9 @@ def run_server_secure(app, cert_file, cert_key): While this function is incomplete, the project is safe to run in production only via a TLS termination proxy with e.g. NGINX. """ + # The chiphers are from the Mozilla project wiki, as a recommendation for + # the most secure and up-to-date build. + # https://wiki.mozilla.org/Security/Server_Side_TLS logger = logging.getLogger("swift-browser-ui") logger.debug("Running server securely.") logger.debug("Setting up SSL context for the server.") From 04fac28483dc86157d3731a67535a50084fcad32 Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 19 Aug 2019 13:30:38 +0300 Subject: [PATCH 18/19] Update ciphers, lock out TLSv1 and TLSv1_1 as deprecated --- swift_browser_ui/server.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/swift_browser_ui/server.py b/swift_browser_ui/server.py index ee210534f..2ddb9a218 100644 --- a/swift_browser_ui/server.py +++ b/swift_browser_ui/server.py @@ -139,17 +139,18 @@ def run_server_secure(app, cert_file, cert_key): logger.debug("Setting up SSL context for the server.") sslcontext = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) cipher_str = ( - 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE' + - '-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-' + - 'AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-' + - 'SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-' + - 'RSA-AES128-SHA256' + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE" + + "-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA" + + "-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM" + + "-SHA256:DHE-RSA-AES256-GCM-SHA384" ) logger.debug( "Setting following ciphers for SSL context: \n%s", cipher_str ) sslcontext.set_ciphers(cipher_str) + sslcontext.options |= ssl.OP_NO_TLSv1 + sslcontext.options |= ssl.OP_NO_TLSv1_1 logger.debug("Loading cert chain.") sslcontext.load_cert_chain(cert_file, cert_key) aiohttp.web.run_app( From 84bd5c06a682c9203beb738e79681fb4e2923252 Mon Sep 17 00:00:00 2001 From: --global Date: Mon, 19 Aug 2019 13:48:09 +0300 Subject: [PATCH 19/19] Bump to v0.3.2 --- docs/source/conf.py | 2 +- swift_browser_ui/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index e6af9b5ae..861c746a2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ author = 'CSC Developers' # The full version, including alpha/beta/rc tags -version = release = '0.3.1' +version = release = '0.3.2' # -- General configuration --------------------------------------------------- diff --git a/swift_browser_ui/__init__.py b/swift_browser_ui/__init__.py index 5690bce12..a1d3d29a4 100644 --- a/swift_browser_ui/__init__.py +++ b/swift_browser_ui/__init__.py @@ -7,6 +7,6 @@ __name__ = 'swift_browser_ui' -__version__ = '0.3.1' +__version__ = '0.3.2' __author__ = 'CSC Developers' __license__ = 'MIT License'