From f31e1716e8e9e41b8cf8f917098ac435edf74fd5 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 15 Apr 2019 14:38:49 -0700 Subject: [PATCH 01/12] Add TABPY_LOG_DETAILS config parameter --- tabpy-server/tabpy_server/app/ConfigParameters.py | 1 + tabpy-server/tabpy_server/app/app.py | 6 ++++++ tabpy-server/tabpy_server/handlers/base_handler.py | 7 +++++++ 3 files changed, 14 insertions(+) diff --git a/tabpy-server/tabpy_server/app/ConfigParameters.py b/tabpy-server/tabpy_server/app/ConfigParameters.py index d333dd16..c6826767 100644 --- a/tabpy-server/tabpy_server/app/ConfigParameters.py +++ b/tabpy-server/tabpy_server/app/ConfigParameters.py @@ -10,3 +10,4 @@ class ConfigParameters: TABPY_CERTIFICATE_FILE = 'TABPY_CERTIFICATE_FILE' TABPY_KEY_FILE = 'TABPY_KEY_FILE' TABPY_PWD_FILE = 'TABPY_PWD_FILE' + TABPY_LOG_DETAILS = 'TABPY_LOG_DETAILS' diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 5539a211..16229711 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -247,6 +247,12 @@ def set_parameter(settings_key, features = self._get_features() self.settings['versions'] = {'v1': {'features': features}} + set_parameter('log_request_context', + ConfigParameters.TABPY_LOG_DETAILS, + default_val='false') + if (self.settings['log_request_context'].lower() != 'false'): + self.settings['log_request_context'] = True + def _validate_transfer_protocol_settings(self): if 'transfer_protocol' not in self.settings: log_and_raise( diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index 621189e1..4b2dd1c4 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -19,6 +19,7 @@ def initialize(self, app): self.port = self.settings['port'] self.python_service = app.python_service self.credentials = app.credentials + self.log_request_context = app.settings['log_request_context'] def error_out(self, code, log_message, info=None): self.set_status(code) @@ -96,3 +97,9 @@ def fail_with_not_authorized(self): 401, info="Unauthorized request.", log_message="Invalid credentials provided.") + + def append_request_context(self, msg): + ''' + Adds request context (caller info) to logged messages. + ''' + pass From 58b2142c3aeac0892e721a7276f3211cfffc761d Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 15 Apr 2019 15:11:18 -0700 Subject: [PATCH 02/12] Add SettingsParameters.py --- .../server_tests/test_service_info_handler.py | 3 +- .../tabpy_server/app/SettingsParameters.py | 13 ++++ tabpy-server/tabpy_server/app/app.py | 73 +++++++++++-------- .../tabpy_server/handlers/base_handler.py | 3 +- .../handlers/management_handler.py | 5 +- .../handlers/service_info_handler.py | 8 +- .../handlers/upload_destination_handler.py | 3 +- tabpy-server/tabpy_server/handlers/util.py | 5 +- tabpy-server/tabpy_server/management/util.py | 5 +- tabpy-server/tabpy_server/psws/callbacks.py | 14 ++-- tabpy-tools/tabpy_tools/rest.py | 2 +- 11 files changed, 83 insertions(+), 51 deletions(-) create mode 100755 tabpy-server/tabpy_server/app/SettingsParameters.py diff --git a/tabpy-server/server_tests/test_service_info_handler.py b/tabpy-server/server_tests/test_service_info_handler.py index 14c8cedf..1f42aa31 100644 --- a/tabpy-server/server_tests/test_service_info_handler.py +++ b/tabpy-server/server_tests/test_service_info_handler.py @@ -2,6 +2,7 @@ import json import os from tabpy_server.app.app import TabPyApp +from tabpy_server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -12,7 +13,7 @@ def _create_expected_info_response(settings, tabpy_state): 'description': tabpy_state.get_description(), 'creation_time': tabpy_state.creation_time, 'state_path': settings['state_file_path'], - 'server_version': settings['server_version'], + 'server_version': settings[SettingsParameters.ServerVersion], 'name': tabpy_state.name, 'versions': settings['versions'] } diff --git a/tabpy-server/tabpy_server/app/SettingsParameters.py b/tabpy-server/tabpy_server/app/SettingsParameters.py new file mode 100755 index 00000000..265d0093 --- /dev/null +++ b/tabpy-server/tabpy_server/app/SettingsParameters.py @@ -0,0 +1,13 @@ +class SettingsParameters: + ''' + Application (TabPyApp) settings names + ''' + TransferProtocol = 'transfer_protocol' + Port = 'port' + ServerVersion = 'server_version' + UploadDir = 'upload_dir' + CertificateFile = 'certificate_file' + KeyFile = 'key_file' + StateFilePath = 'state_file_path' + ApiVersions = 'versions' + LogRequestContext = 'log_request_context' diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 16229711..9614bb23 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -13,6 +13,7 @@ import tabpy_server from tabpy_server import __version__ from tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.app.util import ( log_and_raise, parse_pwd_file) @@ -70,19 +71,19 @@ def run(self): self.tabpy_state, self.python_service) - if self.settings['transfer_protocol'] == 'http': - application.listen(self.settings['port']) - elif self.settings['transfer_protocol'] == 'https': - application.listen(self.settings['port'], + if self.settings[SettingsParameters.TransferProtocol] == 'http': + application.listen(self.settings[SettingsParameters.Port]) + elif self.settings[SettingsParameters.TransferProtocol] == 'https': + application.listen(self.settings[SettingsParameters.Port], ssl_options={ - 'certfile': self.settings['certificate_file'], - 'keyfile': self.settings['key_file'] + 'certfile': self.settings[SettingsParameters.CertificateFile], + 'keyfile': self.settings[SettingsParameters.KeyFile] }) else: log_and_raise('Unsupported transfer protocol.', RuntimeError) logger.info('Web service listening on port {}'.format( - str(self.settings['port']))) + str(self.settings[SettingsParameters.Port]))) tornado.ioloop.IOLoop.instance().start() def _create_tornado_web_app(self): @@ -184,36 +185,41 @@ def set_parameter(settings_key, elif default_val is not None: self.settings[settings_key] = default_val - set_parameter('port', ConfigParameters.TABPY_PORT, + set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT, default_val=9004, check_env_var=True) - set_parameter('server_version', None, default_val=__version__) + set_parameter(SettingsParameters.ServerVersion, None, + default_val=__version__) - set_parameter('upload_dir', ConfigParameters.TABPY_QUERY_OBJECT_PATH, + set_parameter(SettingsParameters.UploadDir, + ConfigParameters.TABPY_QUERY_OBJECT_PATH, default_val='/tmp/query_objects', check_env_var=True) - if not os.path.exists(self.settings['upload_dir']): - os.makedirs(self.settings['upload_dir']) + if not os.path.exists(self.settings[SettingsParameters.UploadDir]): + os.makedirs(self.settings[SettingsParameters.UploadDir]) # set and validate transfer protocol - set_parameter('transfer_protocol', + set_parameter(SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL, default_val='http') - self.settings['transfer_protocol'] =\ - self.settings['transfer_protocol'].lower() + self.settings[SettingsParameters.TransferProtocol] =\ + self.settings[SettingsParameters.TransferProtocol].lower() - set_parameter('certificate_file', + set_parameter(SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE) - set_parameter('key_file', ConfigParameters.TABPY_KEY_FILE) + set_parameter(SettingsParameters.KeyFile, + ConfigParameters.TABPY_KEY_FILE) self._validate_transfer_protocol_settings() # if state.ini does not exist try and create it - remove # last dependence on batch/shell script - set_parameter('state_file_path', ConfigParameters.TABPY_STATE_PATH, + set_parameter(SettingsParameters.StateFilePath, + ConfigParameters.TABPY_STATE_PATH, default_val='./tabpy-server/tabpy_server', check_env_var=True) - self.settings['state_file_path'] = os.path.realpath( + self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( - os.path.expanduser(self.settings['state_file_path']))) - state_file_path = self.settings['state_file_path'] + os.path.expanduser( + self.settings[SettingsParameters.StateFilePath]))) + state_file_path = self.settings[SettingsParameters.StateFilePath] logger.info("Loading state from state file %s" % os.path.join(state_file_path, "state.ini")) tabpy_state = _get_state_from_file(state_file_path) @@ -245,20 +251,22 @@ def set_parameter(settings_key, "Authentication is not enabled") features = self._get_features() - self.settings['versions'] = {'v1': {'features': features}} + self.settings[SettingsParameters.ApiVersions] =\ + {'v1': {'features': features}} - set_parameter('log_request_context', + set_parameter(SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, default_val='false') - if (self.settings['log_request_context'].lower() != 'false'): - self.settings['log_request_context'] = True + if (self.settings[SettingsParameters.LogRequestContext].lower() != + 'false'): + self.settings[SettingsParameters.LogRequestContext] = True def _validate_transfer_protocol_settings(self): - if 'transfer_protocol' not in self.settings: + if SettingsParameters.TransferProtocol not in self.settings: log_and_raise( 'Missing transfer protocol information.', RuntimeError) - protocol = self.settings['transfer_protocol'] + protocol = self.settings[SettingsParameters.TransferProtocol] if protocol == 'http': return @@ -267,16 +275,17 @@ def _validate_transfer_protocol_settings(self): log_and_raise('Unsupported transfer protocol: {}.'.format( protocol), RuntimeError) - self._validate_cert_key_state('The parameter(s) {} must be set.', - 'certificate_file' in self.settings, - 'key_file' in self.settings) - cert = self.settings['certificate_file'] + self._validate_cert_key_state( + 'The parameter(s) {} must be set.', + SettingsParameters.CertificateFile in self.settings, + SettingsParameters.KeyFile in self.settings) + cert = self.settings[SettingsParameters.CertificateFile] self._validate_cert_key_state( 'The parameter(s) {} must point to ' 'an existing file.', os.path.isfile(cert), - os.path.isfile(self.settings['key_file'])) + os.path.isfile(self.settings[SettingsParameters.KeyFile])) tabpy_server.app.util.validate_cert(cert) @staticmethod diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index 4b2dd1c4..e877cff2 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -3,6 +3,7 @@ import json import logging +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers.util import handle_basic_authentication logger = logging.getLogger(__name__) @@ -16,7 +17,7 @@ def initialize(self, app): self.tabpy_state = app.tabpy_state # set content type to application/json self.set_header("Content-Type", "application/json") - self.port = self.settings['port'] + self.port = self.settings[SettingsParameters.Port] self.python_service = app.python_service self.credentials = app.credentials self.log_request_context = app.settings['log_request_context'] diff --git a/tabpy-server/tabpy_server/handlers/management_handler.py b/tabpy-server/tabpy_server/handlers/management_handler.py index 8615c410..ccd914d3 100644 --- a/tabpy-server/tabpy_server/handlers/management_handler.py +++ b/tabpy-server/tabpy_server/handlers/management_handler.py @@ -8,6 +8,7 @@ from tornado import gen +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import MainHandler from tabpy_server.handlers.base_handler import STAGING_THREAD from tabpy_server.management.state import get_query_object_path @@ -42,7 +43,7 @@ def copy_from_local(localpath, remotepath, is_dir=False): class ManagementHandler(MainHandler): def initialize(self, app): super(ManagementHandler, self).initialize(app) - self.port = self.settings['port'] + self.port = self.settings[SettingsParameters.Port] def _get_protocol(self): return 'http://' @@ -94,7 +95,7 @@ def _add_or_update_endpoint(self, action, name, version, request_data): src_path = (request_data['src_path'] if 'src_path' in request_data else None) target_path = get_query_object_path( - self.settings['state_file_path'], name, version) + self.settings[SettingsParameters.StateFilePath], name, version) _path_checker = _compile('^[\\a-zA-Z0-9-_\\s/]+$') # copy from staging if src_path: diff --git a/tabpy-server/tabpy_server/handlers/service_info_handler.py b/tabpy-server/tabpy_server/handlers/service_info_handler.py index 24da7510..1d1c3059 100644 --- a/tabpy-server/tabpy_server/handlers/service_info_handler.py +++ b/tabpy-server/tabpy_server/handlers/service_info_handler.py @@ -1,4 +1,5 @@ import json +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import ManagementHandler @@ -14,8 +15,9 @@ def get(self): info = {} info['description'] = self.tabpy_state.get_description() info['creation_time'] = self.tabpy_state.creation_time - info['state_path'] = self.settings['state_file_path'] - info['server_version'] = self.settings['server_version'] + info['state_path'] = self.settings[SettingsParameters.StateFilePath] + info['server_version'] =\ + self.settings[SettingsParameters.ServerVersion] info['name'] = self.tabpy_state.name - info['versions'] = self.settings['versions'] + info['versions'] = self.settings[SettingsParameters.ApiVersions] self.write(json.dumps(info)) diff --git a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py index c7dff1b0..b31f27aa 100644 --- a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py @@ -1,4 +1,5 @@ import logging +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import ManagementHandler import os @@ -19,6 +20,6 @@ def get(self): self.fail_with_not_authorized() return - path = self.settings['state_file_path'] + path = self.settings[SettingsParameters.StateFilePath] path = os.path.join(path, _QUERY_OBJECT_STAGING_FOLDER) self.write({"path": path}) diff --git a/tabpy-server/tabpy_server/handlers/util.py b/tabpy-server/tabpy_server/handlers/util.py index 58ffde19..76994bb7 100755 --- a/tabpy-server/tabpy_server/handlers/util.py +++ b/tabpy-server/tabpy_server/handlers/util.py @@ -2,6 +2,7 @@ import binascii from hashlib import pbkdf2_hmac import logging +from tabpy_server.app.SettingsParameters import SettingsParameters logger = logging.getLogger(__name__) @@ -157,11 +158,11 @@ def handle_basic_authentication(headers, api_version, settings, credentials): are valid returns True, otherwise False. ''' logger.debug('Handling authentication for request') - if api_version not in settings['versions']: + if api_version not in settings[SettingsParameters.ApiVersions]: logger.critical('Unknown API version "{}"'.format(api_version)) return False - version_settings = settings['versions'][api_version] + version_settings = settings[SettingsParameters.ApiVersions][api_version] if 'features' not in version_settings: logger.info('No features configured for API {}'.format(api_version)) return True diff --git a/tabpy-server/tabpy_server/management/util.py b/tabpy-server/tabpy_server/management/util.py index bdc175d5..9a05fd39 100644 --- a/tabpy-server/tabpy_server/management/util.py +++ b/tabpy-server/tabpy_server/management/util.py @@ -6,6 +6,7 @@ from configparser import ConfigParser as _ConfigParser from datetime import datetime, timedelta, tzinfo from tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.app.util import log_and_raise from time import mktime @@ -13,8 +14,8 @@ def write_state_config(state, settings): - if 'state_file_path' in settings: - state_path = settings['state_file_path'] + if SettingsParameters.StateFilePath in settings: + state_path = settings[SettingsParameters.StateFilePath] else: log_and_raise( '{} is not set'.format( diff --git a/tabpy-server/tabpy_server/psws/callbacks.py b/tabpy-server/tabpy_server/psws/callbacks.py index 2f4f1610..fab69dd2 100644 --- a/tabpy-server/tabpy_server/psws/callbacks.py +++ b/tabpy-server/tabpy_server/psws/callbacks.py @@ -4,6 +4,7 @@ from tornado import gen +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.common.messages import ( LoadObject, DeleteObjects, ListObjects, ObjectList) from tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files @@ -49,7 +50,7 @@ def init_ps_server(settings, tabpy_state): try: object_version = obj_info['version'] get_query_object_path( - settings['state_file_path'], + settings[SettingsParameters.StateFilePath], object_name, object_version) except Exception as e: logger.error('Exception encounted when downloading object: %s' @@ -72,7 +73,7 @@ def init_model_evaluator(settings, tabpy_state, python_service): object_version = obj_info['version'] object_type = obj_info['type'] object_path = get_query_object_path( - settings['state_file_path'], + settings[SettingsParameters.StateFilePath], object_name, object_version) logger.info('Load endpoint: %s, version: %s, type: %s' % @@ -113,7 +114,7 @@ def _get_latest_service_state(settings, endpoint_info['version'] != existing_endpoint['version']: # Either a new endpoint or new endpoint version path_to_new_version = get_query_object_path( - settings['state_file_path'], + settings[SettingsParameters.StateFilePath], endpoint_name, endpoint_info['version']) endpoint_type = endpoint_info.get('type', 'model') diff[endpoint_name] = (endpoint_type, endpoint_info['version'], @@ -137,7 +138,8 @@ def _get_latest_service_state(settings, def on_state_change(settings, tabpy_state, python_service): try: logger.info("Loading state from state file") - config = util._get_state_from_file(settings['state_file_path']) + config = util._get_state_from_file( + settings[SettingsParameters.StateFilePath]) new_ps_state = TabPyState(config=config, settings=settings) (has_changes, changes) = _get_latest_service_state(settings, @@ -158,7 +160,7 @@ def on_state_change(settings, tabpy_state, python_service): python_service.manage_request(DeleteObjects([object_name])) - cleanup_endpoint_files(object_name, settings['upload_dir']) + cleanup_endpoint_files(object_name, settings[SettingsParameters.UploadDir]) else: endpoint_info = new_endpoints[object_name] @@ -177,7 +179,7 @@ def on_state_change(settings, tabpy_state, python_service): # cleanup old version of endpoint files if object_version > 2: cleanup_endpoint_files( - object_name, settings['upload_dir'], [ + object_name, settings[SettingsParameters.UploadDir], [ object_version, object_version - 1]) except Exception as e: diff --git a/tabpy-tools/tabpy_tools/rest.py b/tabpy-tools/tabpy_tools/rest.py index 362071eb..0343a102 100755 --- a/tabpy-tools/tabpy_tools/rest.py +++ b/tabpy-tools/tabpy_tools/rest.py @@ -3,7 +3,7 @@ import requests from requests.auth import HTTPBasicAuth from re import compile -import json +import json as json from collections import MutableMapping as _MutableMapping From d192e3fb9ca13e3f6e33edf0958f2c8a9822998b Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Tue, 16 Apr 2019 10:02:11 -0700 Subject: [PATCH 03/12] Update unit test --- tabpy-server/server_tests/test_config.py | 2 +- tabpy-server/tabpy_server/app/app.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tabpy-server/server_tests/test_config.py b/tabpy-server/server_tests/test_config.py index 452da6bd..58a67b53 100644 --- a/tabpy-server/server_tests/test_config.py +++ b/tabpy-server/server_tests/test_config.py @@ -9,7 +9,6 @@ from unittest.mock import patch, call - def assert_raises_runtime_error(message, fn, args={}): try: fn(*args) @@ -96,6 +95,7 @@ def test_config_file_present(self, mock_os, mock_path_exists, self.assertEqual(app.settings['transfer_protocol'], 'http') self.assertTrue('certificate_file' not in app.settings) self.assertTrue('key_file' not in app.settings) + self.assertEqual(app.settings['log_request_context'], False) os.remove(config_file.name) diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 9614bb23..2cddec7d 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -257,9 +257,9 @@ def set_parameter(settings_key, set_parameter(SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, default_val='false') - if (self.settings[SettingsParameters.LogRequestContext].lower() != - 'false'): - self.settings[SettingsParameters.LogRequestContext] = True + self.settings[SettingsParameters.LogRequestContext] = ( + self.settings[SettingsParameters.LogRequestContext].lower() != + 'false') def _validate_transfer_protocol_settings(self): if SettingsParameters.TransferProtocol not in self.settings: From bc0aa69a001651fad5ed7036f54cd2814ddcf3c3 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Tue, 16 Apr 2019 13:02:58 -0700 Subject: [PATCH 04/12] Merge from dev --- tabpy-server/tabpy_server/app/app.py | 4 +- tabpy-server/tabpy_server/common/default.conf | 4 + .../tabpy_server/handlers/base_handler.py | 232 +++++++++++++++++- .../tabpy_server/handlers/endpoint_handler.py | 12 +- .../handlers/endpoints_handler.py | 9 +- .../handlers/evaluation_plane_handler.py | 7 +- .../handlers/query_plane_handler.py | 18 +- .../handlers/service_info_handler.py | 6 + .../tabpy_server/handlers/status_handler.py | 6 +- .../handlers/upload_destination_handler.py | 4 +- 10 files changed, 270 insertions(+), 32 deletions(-) diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 2cddec7d..e28d13d4 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -26,7 +26,6 @@ EvaluationPlaneHandler, QueryPlaneHandler, ServiceInfoHandler, StatusHandler, UploadDestinationHandler) - from tornado_json.constants import TORNADO_MAJOR @@ -260,6 +259,9 @@ def set_parameter(settings_key, self.settings[SettingsParameters.LogRequestContext] = ( self.settings[SettingsParameters.LogRequestContext].lower() != 'false') + logger.info('Call context logging is {}'.format( + 'enabled' if self.settings[SettingsParameters.LogRequestContext] + else 'disabled')) def _validate_transfer_protocol_settings(self): if SettingsParameters.TransferProtocol not in self.settings: diff --git a/tabpy-server/tabpy_server/common/default.conf b/tabpy-server/tabpy_server/common/default.conf index de0887b6..ad72a812 100755 --- a/tabpy-server/tabpy_server/common/default.conf +++ b/tabpy-server/tabpy_server/common/default.conf @@ -13,6 +13,10 @@ TABPY_STATE_PATH = ./tabpy-server/tabpy_server # TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt # TABPY_KEY_FILE = path/to/key/file.key +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + [loggers] keys=root diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index e877cff2..e100bf73 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -20,7 +20,7 @@ def initialize(self, app): self.port = self.settings[SettingsParameters.Port] self.python_service = app.python_service self.credentials = app.credentials - self.log_request_context = app.settings['log_request_context'] + self.log_request_context = app.settings[SettingsParameters.LogRequestContext] def error_out(self, code, log_message, info=None): self.set_status(code) @@ -49,23 +49,226 @@ def _add_CORS_header(self): origin = self.tabpy_state.get_access_control_allow_origin() if len(origin) > 0: self.set_header("Access-Control-Allow-Origin", origin) - logger.debug("Access-Control-Allow-Origin:{}".format(origin)) + logger.debug(self.append_request_context( + "Access-Control-Allow-Origin:{}".format(origin))) headers = self.tabpy_state.get_access_control_allow_headers() if len(headers) > 0: self.set_header("Access-Control-Allow-Headers", headers) - logger.debug("Access-Control-Allow-Headers:{}".format(headers)) + logger.debug(self.append_request_context( + "Access-Control-Allow-Headers:{}".format(headers))) methods = self.tabpy_state.get_access_control_allow_methods() if len(methods) > 0: self.set_header("Access-Control-Allow-Methods", methods) - logger.debug("Access-Control-Allow-Methods:{}".format(methods)) + logger.debug(self.append_request_context( + "Access-Control-Allow-Methods:{}".format(methods))) def _sanitize_request_data(self, data, keys=KEYS_TO_SANITIZE): """Remove keys so that we can log safely""" for key in keys: data.pop(key, None) + def _get_auth_method(self, api_version) -> (bool, str): + ''' + Finds authentication method if provided. + + Parameters + ---------- + api_version : str + API version for authentication. + + Returns + ------- + bool + True if known authentication method is found. + False otherwise. + + str + Name of authentication method used by client. + If empty no authentication required. + ''' + if api_version not in self.settings[SettingsParameters.ApiVersions]: + logger.critical(f'Unknown API version "{api_version}"') + return False, '' + + version_settings = settings[SettingsParameters.ApiVersions][api_version] + if 'features' not in version_settings: + logger.info(f'No features configured for API "{api_version}"') + return True, '' + + features = version_settings['features'] + if 'authentication' not in features or\ + not features['authentication']['required']: + logger.info( + f'Authentication is not a required feature for API ' + '"{api_version}"') + return True, '' + + auth_feature = features['authentication'] + if 'methods' not in auth_feature: + logger.critical( + f'Authentication method is not configured for API "{api_version}"') + + methods = auth_feature['methods'] + if 'basic-auth' in auth_feature['methods']: + return True, 'basic-auth' + # Add new methods here... + + # No known methods were found + logger.critical( + f'Unknown authentication method(s) "{methods}" are configured ' + 'for API "{api_version}"') + return False, '' + + def _get_basic_auth_credentials(self) -> bool: + ''' + Find credentials for basic access authentication method. Credentials if + found stored in self.username and self.password. + + Returns + ------- + bool + True if valid credentials were found. + False otherwise. + ''' + logger.debug('Checking request headers for authentication data') + if 'Authorization' not in headers: + logger.info('Authorization header not found') + return False + + auth_header = headers['Authorization'] + auth_header_list = headers['Authorization'].split(' ') + if len(auth_header_list) != 2 or\ + auth_header_list[0] != 'Basic': + logger.error(f'Unknown authentication method "{auth_header}"') + return False + + try: + cred = base64.b64decode(auth_header_list[1]).decode('utf-8') + except (binascii.Error, UnicodeDecodeError) as ex: + logger.critical(f'Cannot decode credentials: {str(ex)}') + return False + + login_pwd = cred.split(':') + if len(login_pwd) != 2: + logger.error('Invalid string in encoded credentials') + return False + + self.username = login_pwd[0] + self.password = login_pwd[1] + return True + + def _get_credentials(self, method) -> bool: + ''' + Find credentials for specified authentication method. Credentials if + found stored in self.username and self.password. + + Parameters + ---------- + method: str + Authentication method name. + + Returns + ------- + bool + True if valid credentials were found. + False otherwise. + ''' + if method == 'basic-auth': + return self._get_basic_auth_credentials() + # Add new methods here... + + # No known methods were found + logger.critical( + f'Unknown authentication method(s) "{method}" are configured ' + 'for API "{api_version}"') + return False + + def _validate_basic_auth_credentials(self) -> bool: + ''' + Validates username:pwd if they are the same as + stored credentials. + + Returns + ------- + bool + True if credentials has key login and + credentials[login] equal SHA3(pwd), False + otherwise. + ''' + login = self.username.lower() + logger.debug(f'Validating credentials for user name "{login}"') + if login not in credentials: + logger.error(f'User name "{username}" not found') + return False + + hashed_pwd = hash_password(username, self.password) + if credentials[login].lower() != hashed_pwd.lower(): + logger.error(f'Wrong password for user name "{username}"') + return False + + return True + + def _validate_credentials(self, method) -> bool: + ''' + Validates credentials according to specified methods if they + are what expected. + + Parameters + ---------- + method: str + Authentication method name. + + Returns + ------- + bool + True if credentials are valid. + False otherwise. + ''' + if method == 'basic-auth': + return self._validate_basic_auth_credentials() + # Add new methods here... + + # No known methods were found + logger.critical( + f'Unknown authentication method(s) "{method}" are configured ' + 'for API "{api_version}"') + return False + + + def handle_authentication(self, api_version) -> bool: + ''' + If authentication feature is configured checks provided + credentials. + + Parameters + ---------- + api_version : str + API version for authentication. + + Returns + ------- + bool + True if authentication is not required. + True if authentication is required and valid + credentials provided. + False otherwise. + ''' + logger.debug('Handling authentication') + found, method = _get_auth_method() + if not found: + return False + + if method == '': + # Do not validate credentials + return True + + if not _get_credentials(method): + return False + + return _validate_credentials(method) + def should_fail_with_not_authorized(self): ''' Checks if authentication is required: @@ -79,8 +282,9 @@ def should_fail_with_not_authorized(self): required and validation for credentials passes. True if validation for credentials failed. ''' - logger.debug('Checking if need to handle authentication') - return not handle_basic_authentication( + logger.debug(self.append_request_context( + 'Checking if need to handle authentication')) + return not handle_authentication( self.request.headers, "v1", self.settings, @@ -90,7 +294,8 @@ def fail_with_not_authorized(self): ''' Prepares server 401 response. ''' - logger.error('Failing with 401 for unauthorized request') + logger.error(self.append_request_context( + 'Failing with 401 for unauthorized request')) self.set_status(401) self.set_header('WWW-Authenticate', 'Basic realm="{}"'.format(self.tabpy_state.name)) @@ -99,8 +304,17 @@ def fail_with_not_authorized(self): info="Unauthorized request.", log_message="Invalid credentials provided.") - def append_request_context(self, msg): + def append_request_context(self, msg) -> str: ''' Adds request context (caller info) to logged messages. ''' - pass + context = '' + if self.log_request_context: + # log request details + context = f'{self.request.remote_ip} calls {self.request.method} {self.request.full_url()}\n' + if 'TabPy-Client' in self.request.headers: + context += f'Client: {self.request.headers["TabPy-Client"]}\n' + if 'TabPy-User' in self.request.headers: + context += f'Tableau user: {self.request.headers["TabPy-User"]}\n' + + return context + msg diff --git a/tabpy-server/tabpy_server/handlers/endpoint_handler.py b/tabpy-server/tabpy_server/handlers/endpoint_handler.py index 5a2e0fd3..c6a6ab47 100644 --- a/tabpy-server/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoint_handler.py @@ -27,7 +27,8 @@ def initialize(self, app): super(EndpointHandler, self).initialize(app) def get(self, endpoint_name): - logger.debug('Processing GET for /endpoints/{}'.format(endpoint_name)) + logger.debug(self.append_request_context( + f'Processing GET for /endpoints/{endpoint_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -46,7 +47,8 @@ def get(self, endpoint_name): @tornado.web.asynchronous @gen.coroutine def put(self, name): - logger.debug('Processing PUT for /endpoints/{}'.format(name)) + logger.debug(self.append_request_context( + f'Processing PUT for /endpoints/{name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -76,7 +78,8 @@ def put(self, name): return new_version = int(endpoints[name]['version']) + 1 - logger.info('Endpoint info: %s' % request_data) + logger.info(self.append_request_context( + 'Endpoint info: %s' % request_data)) err_msg = yield self._add_or_update_endpoint( 'update', name, new_version, request_data) if err_msg: @@ -94,7 +97,8 @@ def put(self, name): @tornado.web.asynchronous @gen.coroutine def delete(self, name): - logger.debug('Processing DELETE for /endpoints/{}'.format(name)) + logger.debug(self.append_request_context( + 'Processing DELETE for /endpoints/{}'.format(name))) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return diff --git a/tabpy-server/tabpy_server/handlers/endpoints_handler.py b/tabpy-server/tabpy_server/handlers/endpoints_handler.py index b18c8a8b..8604df15 100644 --- a/tabpy-server/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoints_handler.py @@ -32,7 +32,8 @@ def get(self): @tornado.web.asynchronous @gen.coroutine def post(self): - logger.debug('Processing POST for /endpoints') + logger.debug(self.append_request_context( + 'Processing POST for /endpoints')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -68,13 +69,15 @@ def post(self): self.finish() return - logger.debug("Adding endpoint '{}'".format(name)) + logger.debug(self.append_request_context( + "Adding endpoint '{}'".format(name))) err_msg = yield self._add_or_update_endpoint('add', name, 1, request_data) if err_msg: self.error_out(400, err_msg) else: - logger.debug("Endpoint {} successfully added".format(name)) + logger.debug(self.append_request_context( + "Endpoint {} successfully added".format(name))) self.set_status(201) self.write(self.tabpy_state.get_endpoints(name)) self.finish() diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py index a48fb4b2..22767c85 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py @@ -39,7 +39,8 @@ def initialize(self, executor, app): @tornado.web.asynchronous @gen.coroutine def post(self): - logger.debug('Processing POST for /evaluate') + logger.debug(self.append_request_context( + 'Processing POST for /evaluate')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -80,8 +81,8 @@ def post(self): for u in user_code.splitlines(): function_to_evaluate += ' ' + u + '\n' - logger.info( - "function to evaluate=%s" % function_to_evaluate) + logger.info(self.append_request_context( + f'function to evaluate={function_to_evaluate}')) result = yield self.call_subprocess(function_to_evaluate, arguments) diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy-server/tabpy_server/handlers/query_plane_handler.py index e0ad556f..04cd1245 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/query_plane_handler.py @@ -71,14 +71,16 @@ def _query(self, po_name, data, uid, qry): 'utf-8')).hexdigest()) return (QuerySuccessful, response.for_json(), gls_time) else: - logger.error("Failed query, response: {}".format(response)) + logger.error(self.append_request_context( + f'Failed query, response: {response}')) return (type(response), response.for_json(), gls_time) # handle HTTP Options requests to support CORS # don't check API key (client does not send or receive data for OPTIONS, # it just allows the client to subsequently make a POST request) def options(self, pred_name): - logger.debug('Processing OPTIONS for /query/{}'.format(pred_name)) + logger.debug(self.append_request_context( + f'Processing OPTIONS for /query/{pred_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -150,8 +152,8 @@ def _process_query(self, endpoint_name, start): return if po_name != endpoint_name: - logger.info( - "Querying actual model: po_name={}".format(po_name)) + logger.info(self.append_request_context( + f'Querying actual model: po_name={po_name}')) uid = _get_uuid() @@ -198,7 +200,8 @@ def _get_actual_model(self, endpoint_name): @tornado.web.asynchronous def get(self, endpoint_name): - logger.debug('Processing GET for /query/{}'.format(endpoint_name)) + logger.debug(self.append_request_context( + f'Processing GET for /query/{endpoint_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -208,12 +211,12 @@ def get(self, endpoint_name): endpoint_name = urllib.parse.unquote(endpoint_name) else: endpoint_name = urllib.unquote(endpoint_name) - logger.debug("GET /query/{}".format(endpoint_name)) self._process_query(endpoint_name, start) @tornado.web.asynchronous def post(self, endpoint_name): - logger.debug('Processing POST for /query/{}'.format(endpoint_name)) + logger.debug(self.append_request_context( + f'Processing POST for /query/{endpoint_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return @@ -223,5 +226,4 @@ def post(self, endpoint_name): endpoint_name = urllib.parse.unquote(endpoint_name) else: endpoint_name = urllib.unquote(endpoint_name) - logger.debug("POST /query/{}".format(endpoint_name)) self._process_query(endpoint_name, start) diff --git a/tabpy-server/tabpy_server/handlers/service_info_handler.py b/tabpy-server/tabpy_server/handlers/service_info_handler.py index 1d1c3059..74fbc8e8 100644 --- a/tabpy-server/tabpy_server/handlers/service_info_handler.py +++ b/tabpy-server/tabpy_server/handlers/service_info_handler.py @@ -1,8 +1,12 @@ import json +import logging from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import ManagementHandler +logger = logging.getLogger(__name__) + + class ServiceInfoHandler(ManagementHandler): def initialize(self, app): super(ServiceInfoHandler, self).initialize(app) @@ -11,6 +15,8 @@ def get(self): # do not check for authentication - this method # is the only way for client to collect info about # supported API versions and required features + logger.debug(self.append_request_context( + 'Processing GET for /info')) self._add_CORS_header() info = {} info['description'] = self.tabpy_state.get_description() diff --git a/tabpy-server/tabpy_server/handlers/status_handler.py b/tabpy-server/tabpy_server/handlers/status_handler.py index eccd67fe..4297f98f 100644 --- a/tabpy-server/tabpy_server/handlers/status_handler.py +++ b/tabpy-server/tabpy_server/handlers/status_handler.py @@ -17,7 +17,8 @@ def get(self): self._add_CORS_header() - logger.debug("Obtaining service status") + logger.debug(self.append_request_context( + "Obtaining service status")) status_dict = {} for k, v in self.python_service.ps.query_objects.items(): status_dict[k] = { @@ -26,7 +27,8 @@ def get(self): 'status': v['status'], 'last_error': v['last_error']} - logger.debug("Found models: {}".format(status_dict)) + logger.debug(self.append_request_context( + f'Found models: {status_dict}')) self.write(json.dumps(status_dict)) self.finish() return diff --git a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py index b31f27aa..5090101e 100644 --- a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py @@ -14,8 +14,8 @@ def initialize(self, app): super(UploadDestinationHandler, self).initialize(app) def get(self): - logger.debug( - 'Processing GET for /configurations/endpoint_upload_destination') + logger.debug(self.append_request_context( + 'Processing GET for /configurations/endpoint_upload_destination')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return From 2e12171104bf47dfb2dac4fbcb06264a5aaefecb Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 17 Apr 2019 12:02:26 -0700 Subject: [PATCH 05/12] Fix static pages path --- .../tabpy_server/app/ConfigParameters.py | 1 + .../tabpy_server/app/SettingsParameters.py | 1 + tabpy-server/tabpy_server/app/app.py | 11 +- tabpy-server/tabpy_server/common/default.conf | 3 + .../tabpy_server/handlers/__init__.py | 1 - .../tabpy_server/handlers/base_handler.py | 34 ++-- tabpy-server/tabpy_server/handlers/util.py | 148 ------------------ 7 files changed, 28 insertions(+), 171 deletions(-) diff --git a/tabpy-server/tabpy_server/app/ConfigParameters.py b/tabpy-server/tabpy_server/app/ConfigParameters.py index c6826767..0af2c5af 100644 --- a/tabpy-server/tabpy_server/app/ConfigParameters.py +++ b/tabpy-server/tabpy_server/app/ConfigParameters.py @@ -11,3 +11,4 @@ class ConfigParameters: TABPY_KEY_FILE = 'TABPY_KEY_FILE' TABPY_PWD_FILE = 'TABPY_PWD_FILE' TABPY_LOG_DETAILS = 'TABPY_LOG_DETAILS' + TABPY_STATIC_PATH = 'TABPY_STATIC_PATH' diff --git a/tabpy-server/tabpy_server/app/SettingsParameters.py b/tabpy-server/tabpy_server/app/SettingsParameters.py index 265d0093..b070a2b4 100755 --- a/tabpy-server/tabpy_server/app/SettingsParameters.py +++ b/tabpy-server/tabpy_server/app/SettingsParameters.py @@ -11,3 +11,4 @@ class SettingsParameters: StateFilePath = 'state_file_path' ApiVersions = 'versions' LogRequestContext = 'log_request_context' + StaticPath = 'static_path' diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index e28d13d4..9bb44529 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -118,8 +118,7 @@ def _create_tornado_web_app(self): dict(app=self)), (self.subdirectory + r'/(.*)', tornado.web.StaticFileHandler, dict(path=self.settings['static_path'], - default_filename="index.html", - app=self)), + default_filename="index.html")), ], debug=False, **self.settings) return application @@ -228,8 +227,12 @@ def set_parameter(settings_key, self.python_service = PythonServiceHandler(PythonService()) self.settings['compress_response'] = True if TORNADO_MAJOR >= 4\ else "gzip" - self.settings['static_path'] = os.path.join( - os.path.dirname(__file__), "static") + + set_parameter(SettingsParameters.StaticPath, + ConfigParameters.TABPY_STATIC_PATH) + self.settings[SettingsParameters.StaticPath] =\ + os.path.abspath(self.settings[SettingsParameters.StaticPath]) + logger.debug(f'Static pages folder set to "{self.settings[SettingsParameters.StaticPath]}"') # Set subdirectory from config if applicable if tabpy_state.has_option("Service Info", "Subdirectory"): diff --git a/tabpy-server/tabpy_server/common/default.conf b/tabpy-server/tabpy_server/common/default.conf index ad72a812..6eef5265 100755 --- a/tabpy-server/tabpy_server/common/default.conf +++ b/tabpy-server/tabpy_server/common/default.conf @@ -3,6 +3,9 @@ TABPY_QUERY_OBJECT_PATH = /tmp/query_objects TABPY_PORT = 9004 TABPY_STATE_PATH = ./tabpy-server/tabpy_server +# Where static pages live +TABPY_STATIC_PATH = ./tabpy-server/tabpy_server/static + # For how to configure TabPy authentication read # docs/server-config.md. # TABPY_PWD_FILE = /path/to/password/file.txt diff --git a/tabpy-server/tabpy_server/handlers/__init__.py b/tabpy-server/tabpy_server/handlers/__init__.py index 59af73b7..7dc480b4 100644 --- a/tabpy-server/tabpy_server/handlers/__init__.py +++ b/tabpy-server/tabpy_server/handlers/__init__.py @@ -11,4 +11,3 @@ from tabpy_server.handlers.status_handler import StatusHandler from tabpy_server.handlers.upload_destination_handler\ import UploadDestinationHandler -from tabpy_server.handlers.util import handle_basic_authentication diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index e100bf73..b87a17d5 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -1,10 +1,12 @@ +import base64 +import binascii import concurrent import tornado.web import json import logging from tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy_server.handlers.util import handle_basic_authentication +from tabpy_server.handlers.util import hash_password logger = logging.getLogger(__name__) STAGING_THREAD = concurrent.futures.ThreadPoolExecutor(max_workers=3) @@ -92,7 +94,7 @@ def _get_auth_method(self, api_version) -> (bool, str): logger.critical(f'Unknown API version "{api_version}"') return False, '' - version_settings = settings[SettingsParameters.ApiVersions][api_version] + version_settings = self.settings[SettingsParameters.ApiVersions][api_version] if 'features' not in version_settings: logger.info(f'No features configured for API "{api_version}"') return True, '' @@ -133,12 +135,12 @@ def _get_basic_auth_credentials(self) -> bool: False otherwise. ''' logger.debug('Checking request headers for authentication data') - if 'Authorization' not in headers: + if 'Authorization' not in self.request.headers: logger.info('Authorization header not found') return False - auth_header = headers['Authorization'] - auth_header_list = headers['Authorization'].split(' ') + auth_header = self.request.headers['Authorization'] + auth_header_list = auth_header.split(' ') if len(auth_header_list) != 2 or\ auth_header_list[0] != 'Basic': logger.error(f'Unknown authentication method "{auth_header}"') @@ -199,13 +201,13 @@ def _validate_basic_auth_credentials(self) -> bool: ''' login = self.username.lower() logger.debug(f'Validating credentials for user name "{login}"') - if login not in credentials: - logger.error(f'User name "{username}" not found') + if login not in self.credentials: + logger.error(f'User name "{self.username}" not found') return False - hashed_pwd = hash_password(username, self.password) - if credentials[login].lower() != hashed_pwd.lower(): - logger.error(f'Wrong password for user name "{username}"') + hashed_pwd = hash_password(self.username, self.password) + if self.credentials[login].lower() != hashed_pwd.lower(): + logger.error(f'Wrong password for user name "{self.username}"') return False return True @@ -256,7 +258,7 @@ def handle_authentication(self, api_version) -> bool: False otherwise. ''' logger.debug('Handling authentication') - found, method = _get_auth_method() + found, method = self._get_auth_method(api_version) if not found: return False @@ -264,10 +266,10 @@ def handle_authentication(self, api_version) -> bool: # Do not validate credentials return True - if not _get_credentials(method): + if not self._get_credentials(method): return False - return _validate_credentials(method) + return self._validate_credentials(method) def should_fail_with_not_authorized(self): ''' @@ -284,11 +286,7 @@ def should_fail_with_not_authorized(self): ''' logger.debug(self.append_request_context( 'Checking if need to handle authentication')) - return not handle_authentication( - self.request.headers, - "v1", - self.settings, - self.credentials) + return not self.handle_authentication("v1") def fail_with_not_authorized(self): ''' diff --git a/tabpy-server/tabpy_server/handlers/util.py b/tabpy-server/tabpy_server/handlers/util.py index 76994bb7..99617d87 100755 --- a/tabpy-server/tabpy_server/handlers/util.py +++ b/tabpy-server/tabpy_server/handlers/util.py @@ -36,151 +36,3 @@ def hash_password(username, pwd): salt=salt.encode(), iterations=10000) return binascii.hexlify(hash).decode() - - -def validate_basic_auth_credentials(username, pwd, credentials): - ''' - Validates username:pwd if they are the same as - stored credentials. - - Parameters - ---------- - username : str - User name (login). - - pwd : str - Password in plain text. The function will hash - the password with SHA3 to compare with hashed - passwords in stored credentials. - - credentials: dict - Dictionary of stored credentials where keys are - user names and values are SHA3-hashed passwords. - - Returns - ------- - bool - True if credentials has key login and - credentials[login] equal SHA3(pwd), False - otherwise. - ''' - login = username.lower() - logger.debug('Validating credentials for user name "{}"'.format(login)) - if login not in credentials: - logger.error('User name "{}" not found'.format(username)) - return False - - hashed_pwd = hash_password(username, pwd) - if credentials[login].lower() != hashed_pwd.lower(): - logger.error('Wrong password for user name "{}"'.format(username)) - return False - - return True - - -def check_and_validate_basic_auth_credentials(headers, credentials): - ''' - Checks if credentials are provided in headers and - if they are valid. - - Parameters - ---------- - headers - HTTP request headers. - - credentials: dict - Dictionary of stored credentials where keys are - user names and values are SHA3-hashed passwords. - - Returns - ------- - bool - True if credentials are present in headers and - they are valid. - ''' - logger.debug('Checking request headers for authentication data') - if 'Authorization' not in headers: - logger.info('Authorization header not found') - return False - - auth_header = headers['Authorization'] - auth_header_list = headers['Authorization'].split(' ') - if len(auth_header_list) != 2 or\ - auth_header_list[0] != 'Basic': - logger.error('Unknown authentication method "{}"'.format(auth_header)) - return False - - try: - cred = base64.b64decode(auth_header_list[1]).decode('utf-8') - except (binascii.Error, UnicodeDecodeError) as ex: - logger.critical('Cannot decode credentials: {}'.format(str(ex))) - return False - - login_pwd = cred.split(':') - if len(login_pwd) != 2: - logger.error('Invalid string in encoded credentials') - return False - - return validate_basic_auth_credentials(login_pwd[0], - login_pwd[1], - credentials) - - -def handle_basic_authentication(headers, api_version, settings, credentials): - ''' - Checks if credentials need to be validated and they are - validates them. - - Parameters - ---------- - headers - HTTP request headers. - - api_version : str - API version for authentication. - - settings : dict - Application settings (TabPyApp.settings). - - credentials: dict - Dictionary of stored credentials where keys are - user names and values are SHA3-hashed passwords. - - Returns - ------- - bool - If for the specified API version authentication is - not turned on returns True. - Otherwise checks what authentication type is used - and if it is supported type validates provided - credentials. - If authentication type is supported and credentials - are valid returns True, otherwise False. - ''' - logger.debug('Handling authentication for request') - if api_version not in settings[SettingsParameters.ApiVersions]: - logger.critical('Unknown API version "{}"'.format(api_version)) - return False - - version_settings = settings[SettingsParameters.ApiVersions][api_version] - if 'features' not in version_settings: - logger.info('No features configured for API {}'.format(api_version)) - return True - - features = version_settings['features'] - if 'authentication' not in features or\ - not features['authentication']['required']: - logger.info( - 'Authentication is not a required feature for API ' - '{}'.format(api_version)) - return True - - auth_feature = features['authentication'] - if 'methods' not in auth_feature or\ - 'basic-auth' not in auth_feature['methods']: - logger.critical( - 'Basic authentication access method is not configured ' - 'for API {}'.format(api_version)) - return False - - return check_and_validate_basic_auth_credentials(headers, credentials) From 0e51d5eeb00b30b6f41a860f6627f2aba91eb632 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 17 Apr 2019 14:56:03 -0700 Subject: [PATCH 06/12] Remove duplicated tests --- .../test_evaluation_plane_handler.py | 2 +- .../server_tests/test_validate_credentials.py | 198 ------------------ tabpy-server/tabpy_server/app/app.py | 5 +- 3 files changed, 4 insertions(+), 201 deletions(-) delete mode 100755 tabpy-server/server_tests/test_validate_credentials.py diff --git a/tabpy-server/server_tests/test_evaluation_plane_handler.py b/tabpy-server/server_tests/test_evaluation_plane_handler.py index 5e9a57e2..2f9ad4a6 100755 --- a/tabpy-server/server_tests/test_evaluation_plane_handler.py +++ b/tabpy-server/server_tests/test_evaluation_plane_handler.py @@ -128,7 +128,7 @@ def test_valid_creds_pass(self): def test_null_request(self): response = self.fetch('') - self.assertEqual(599, response.code) + self.assertEqual(404, response.code) def test_script_not_present(self): response = self.fetch( diff --git a/tabpy-server/server_tests/test_validate_credentials.py b/tabpy-server/server_tests/test_validate_credentials.py deleted file mode 100755 index 028c5409..00000000 --- a/tabpy-server/server_tests/test_validate_credentials.py +++ /dev/null @@ -1,198 +0,0 @@ -import base64 -import hashlib -import logging -import pathlib -import os -import unittest -from argparse import Namespace -from tempfile import NamedTemporaryFile - -from tabpy_server.handlers.util import ( - validate_basic_auth_credentials, - handle_basic_authentication, - check_and_validate_basic_auth_credentials) - -from unittest.mock import patch, call - - -class TestValidateBasicAuthCredentials(unittest.TestCase): - def setUp(self): - self.credentials = { - # PBKDF2('user1', 'password', 10000) - 'user1': - ('5961c87343553bf078add1189b4f59238f663eabd23dbc1dc538c5fed' - 'f18a8cc546f4a46500dd7672144595ac4e9610dc9edc66ee1cb7b58cab' - '64ddb662390b3') - } - - def test_given_unknown_username_expect_validation_fails(self): - self.assertFalse(validate_basic_auth_credentials( - 'user2', 'pwd@2', self.credentials)) - - def test_given_wrong_password_expect_validation_fails(self): - self.assertFalse(validate_basic_auth_credentials( - 'user1', 'password#1', self.credentials)) - - def test_given_valid_creds_expect_validation_passes(self): - self.assertTrue(validate_basic_auth_credentials( - 'user1', 'password', self.credentials)) - - def test_given_valid_creds_mixcase_login_expect_validation_passes(self): - self.assertTrue(validate_basic_auth_credentials( - 'UsEr1', 'password', self.credentials)) - - -class TestCheckAndValidateBasicAuthCredentials(unittest.TestCase): - def setUp(self): - self.credentials = { - # PBKDF2('user1', 'password', 10000) - 'user1': - ('5961c87343553bf078add1189b4f59238f663eabd23dbc1dc538c5fed' - 'f18a8cc546f4a46500dd7672144595ac4e9610dc9edc66ee1cb7b58cab' - '64ddb662390b3') - } - - def test_given_no_headers_expect_validation_fails(self): - self.assertFalse( - check_and_validate_basic_auth_credentials({}, self.credentials)) - - def test_given_bad_auth_header_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Some unexpected string' - }, self.credentials)) - - def test_given_bad_encoded_credentials_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic abc' - }, self.credentials)) - - def test_given_malformed_credentials_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format( - base64.b64encode('user1-password'.encode('utf-8')). - decode('utf-8')) - }, self.credentials)) - - def test_given_unknown_username_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format( - base64.b64encode('unknown_user:password'.encode('utf-8'))) - }, self.credentials)) - - def test_given_wrong_pwd_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format( - base64.b64encode('user1:p@ssw0rd'.encode('utf-8'))) - }, self.credentials)) - - def test_given_valid_creds_expect_validation_passes(self): - b64_username_pwd = base64.b64encode( - 'user1:password'.encode('utf-8')).decode('utf-8') - self.assertTrue(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format(b64_username_pwd) - }, self.credentials)) - - -class TestHandleAuthentication(unittest.TestCase): - def setUp(self): - self.credentials = { - # PBKDF2('user1', 'password', 10000) - 'user1': - ('5961c87343553bf078add1189b4f59238f663eabd23dbc1dc538c5fed' - 'f18a8cc546f4a46500dd7672144595ac4e9610dc9edc66ee1cb7b58cab' - '64ddb662390b3') - } - - self.settings = { - 'versions': - { - 'v0.1a': - { - 'features': {} - }, - 'v0.2beta': - { - 'features': - { - 'authentication': - { - 'required': True, - } - } - }, - 'v0.3gamma': - { - 'features': - { - 'authentication': - { - 'required': True, - 'methods': - { - 'unknown-auth': {} - } - } - } - }, - 'v0.4yota': {}, - 'v1': - { - 'features': - { - 'authentication': - { - 'required': True, - 'methods': - { - 'basic-auth': {} - } - } - } - } - } - } - - def test_given_no_api_version_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, '', self.settings, self.credentials)) - - def test_given_unknown_api_version_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v0.314p', self.settings, self.credentials)) - - def test_given_auth_is_not_configured_expect_success(self): - self.assertTrue(handle_basic_authentication( - {}, 'v0.1a', self.settings, self.credentials)) - - def test_given_auth_method_not_provided_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v0.2beta', self.settings, self.credentials)) - - def test_given_auth_method_is_unknown_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v0.3gamma', self.settings, self.credentials)) - - def test_given_features_not_configured_expect_success(self): - self.assertTrue(handle_basic_authentication( - {}, 'v0.4yota', self.settings, self.credentials)) - - def test_given_headers_not_provided_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v1', self.settings, self.credentials)) - - def test_given_valid_creds_expect_success(self): - b64_username_pwd = base64.b64encode( - 'user1:password'.encode('utf-8')).decode('utf-8') - self.assertTrue(handle_basic_authentication( - { - 'Authorization': 'Basic {}'.format(b64_username_pwd) - }, - 'v1', - self.settings, - self.credentials)) diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 9bb44529..25dd986b 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -117,7 +117,7 @@ def _create_tornado_web_app(self): UploadDestinationHandler, dict(app=self)), (self.subdirectory + r'/(.*)', tornado.web.StaticFileHandler, - dict(path=self.settings['static_path'], + dict(path=self.settings[SettingsParameters.StaticPath], default_filename="index.html")), ], debug=False, **self.settings) @@ -229,7 +229,8 @@ def set_parameter(settings_key, else "gzip" set_parameter(SettingsParameters.StaticPath, - ConfigParameters.TABPY_STATIC_PATH) + ConfigParameters.TABPY_STATIC_PATH, + default_val='./') self.settings[SettingsParameters.StaticPath] =\ os.path.abspath(self.settings[SettingsParameters.StaticPath]) logger.debug(f'Static pages folder set to "{self.settings[SettingsParameters.StaticPath]}"') From fecba8a1e09434cf491f0612b3f6ab1bf3f6ee3b Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Wed, 17 Apr 2019 15:35:19 -0700 Subject: [PATCH 07/12] Fix styling --- tabpy-server/tabpy_server/app/app.py | 3 ++- .../tabpy_server/common/endpoint_file_mgr.py | 2 +- .../tabpy_server/handlers/base_handler.py | 24 ++++++++++++------- .../tabpy_server/handlers/endpoint_handler.py | 15 +++++++----- .../handlers/endpoints_handler.py | 5 ++-- .../handlers/evaluation_plane_handler.py | 5 ++-- .../handlers/query_plane_handler.py | 15 +++++++----- .../handlers/upload_destination_handler.py | 5 ++-- tabpy-server/tabpy_server/management/state.py | 1 - tabpy-server/tabpy_server/psws/callbacks.py | 3 ++- tabpy-tools/tabpy_tools/rest.py | 6 +++-- tabpy-tools/tools_tests/test_rest.py | 3 ++- 12 files changed, 54 insertions(+), 33 deletions(-) diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 25dd986b..94d525af 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -233,7 +233,8 @@ def set_parameter(settings_key, default_val='./') self.settings[SettingsParameters.StaticPath] =\ os.path.abspath(self.settings[SettingsParameters.StaticPath]) - logger.debug(f'Static pages folder set to "{self.settings[SettingsParameters.StaticPath]}"') + logger.debug(f'Static pages folder set to ' + '"{self.settings[SettingsParameters.StaticPath]}"') # Set subdirectory from config if applicable if tabpy_state.has_option("Service Info", "Subdirectory"): diff --git a/tabpy-server/tabpy_server/common/endpoint_file_mgr.py b/tabpy-server/tabpy_server/common/endpoint_file_mgr.py index d46d1dfd..d1b79beb 100644 --- a/tabpy-server/tabpy_server/common/endpoint_file_mgr.py +++ b/tabpy-server/tabpy_server/common/endpoint_file_mgr.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) -_name_checker = _compile('^[a-zA-Z0-9-_\ ]+$') +_name_checker = _compile(r'^[a-zA-Z0-9-_\s]+$') def _check_endpoint_name(name): diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index b87a17d5..32ca5dc3 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -22,7 +22,9 @@ def initialize(self, app): self.port = self.settings[SettingsParameters.Port] self.python_service = app.python_service self.credentials = app.credentials - self.log_request_context = app.settings[SettingsParameters.LogRequestContext] + self.log_request_context =\ + app.settings[SettingsParameters.LogRequestContext] + self.username = None def error_out(self, code, log_message, info=None): self.set_status(code) @@ -94,7 +96,8 @@ def _get_auth_method(self, api_version) -> (bool, str): logger.critical(f'Unknown API version "{api_version}"') return False, '' - version_settings = self.settings[SettingsParameters.ApiVersions][api_version] + version_settings =\ + self.settings[SettingsParameters.ApiVersions][api_version] if 'features' not in version_settings: logger.info(f'No features configured for API "{api_version}"') return True, '' @@ -110,7 +113,8 @@ def _get_auth_method(self, api_version) -> (bool, str): auth_feature = features['authentication'] if 'methods' not in auth_feature: logger.critical( - f'Authentication method is not configured for API "{api_version}"') + f'Authentication method is not configured for API ' + '"{api_version}"') methods = auth_feature['methods'] if 'basic-auth' in auth_feature['methods']: @@ -238,7 +242,6 @@ def _validate_credentials(self, method) -> bool: 'for API "{api_version}"') return False - def handle_authentication(self, api_version) -> bool: ''' If authentication feature is configured checks provided @@ -265,7 +268,7 @@ def handle_authentication(self, api_version) -> bool: if method == '': # Do not validate credentials return True - + if not self._get_credentials(method): return False @@ -309,10 +312,15 @@ def append_request_context(self, msg) -> str: context = '' if self.log_request_context: # log request details - context = f'{self.request.remote_ip} calls {self.request.method} {self.request.full_url()}\n' + context = (f'{self.request.remote_ip} calls ' + '{self.request.method} {self.request.full_url()}') if 'TabPy-Client' in self.request.headers: - context += f'Client: {self.request.headers["TabPy-Client"]}\n' + context += f', Client: {self.request.headers["TabPy-Client"]}' if 'TabPy-User' in self.request.headers: - context += f'Tableau user: {self.request.headers["TabPy-User"]}\n' + context +=\ + f', Tableau user: {self.request.headers["TabPy-User"]}' + if self.username is not None and self.username != '': + context += f', TabPy user: {self.username}' + context += '\n' return context + msg diff --git a/tabpy-server/tabpy_server/handlers/endpoint_handler.py b/tabpy-server/tabpy_server/handlers/endpoint_handler.py index c6a6ab47..f5a69a9d 100644 --- a/tabpy-server/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoint_handler.py @@ -27,12 +27,13 @@ def initialize(self, app): super(EndpointHandler, self).initialize(app) def get(self, endpoint_name): - logger.debug(self.append_request_context( - f'Processing GET for /endpoints/{endpoint_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing GET for /endpoints/{endpoint_name}')) + self._add_CORS_header() if not endpoint_name: self.write(json.dumps(self.tabpy_state.get_endpoints())) @@ -47,12 +48,13 @@ def get(self, endpoint_name): @tornado.web.asynchronous @gen.coroutine def put(self, name): - logger.debug(self.append_request_context( - f'Processing PUT for /endpoints/{name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing PUT for /endpoints/{name}')) + try: if not self.request.body: self.error_out(400, "Input body cannot be empty") @@ -97,12 +99,13 @@ def put(self, name): @tornado.web.asynchronous @gen.coroutine def delete(self, name): - logger.debug(self.append_request_context( - 'Processing DELETE for /endpoints/{}'.format(name))) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing DELETE for /endpoints/{}'.format(name))) + try: endpoints = self.tabpy_state.get_endpoints(name) if len(endpoints) == 0: diff --git a/tabpy-server/tabpy_server/handlers/endpoints_handler.py b/tabpy-server/tabpy_server/handlers/endpoints_handler.py index 8604df15..88557cea 100644 --- a/tabpy-server/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoints_handler.py @@ -32,12 +32,13 @@ def get(self): @tornado.web.asynchronous @gen.coroutine def post(self): - logger.debug(self.append_request_context( - 'Processing POST for /endpoints')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing POST for /endpoints')) + try: if not self.request.body: self.error_out(400, "Input body cannot be empty") diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py index 22767c85..222d32b0 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py @@ -39,12 +39,13 @@ def initialize(self, executor, app): @tornado.web.asynchronous @gen.coroutine def post(self): - logger.debug(self.append_request_context( - 'Processing POST for /evaluate')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing POST for /evaluate')) + self._add_CORS_header() try: body = json.loads(self.request.body.decode('utf-8')) diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy-server/tabpy_server/handlers/query_plane_handler.py index 04cd1245..a28a63fd 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/query_plane_handler.py @@ -79,12 +79,13 @@ def _query(self, po_name, data, uid, qry): # don't check API key (client does not send or receive data for OPTIONS, # it just allows the client to subsequently make a POST request) def options(self, pred_name): - logger.debug(self.append_request_context( - f'Processing OPTIONS for /query/{pred_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing OPTIONS for /query/{pred_name}')) + # add CORS headers if TabPy has a cors_origin specified self._add_CORS_header() self.write({}) @@ -200,12 +201,13 @@ def _get_actual_model(self, endpoint_name): @tornado.web.asynchronous def get(self, endpoint_name): - logger.debug(self.append_request_context( - f'Processing GET for /query/{endpoint_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing GET for /query/{endpoint_name}')) + start = time.time() if sys.version_info > (3, 0): endpoint_name = urllib.parse.unquote(endpoint_name) @@ -215,12 +217,13 @@ def get(self, endpoint_name): @tornado.web.asynchronous def post(self, endpoint_name): - logger.debug(self.append_request_context( - f'Processing POST for /query/{endpoint_name}')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing POST for /query/{endpoint_name}')) + start = time.time() if sys.version_info > (3, 0): endpoint_name = urllib.parse.unquote(endpoint_name) diff --git a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py index 5090101e..33a559f1 100644 --- a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py @@ -14,12 +14,13 @@ def initialize(self, app): super(UploadDestinationHandler, self).initialize(app) def get(self): - logger.debug(self.append_request_context( - 'Processing GET for /configurations/endpoint_upload_destination')) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing GET for /configurations/endpoint_upload_destination')) + path = self.settings[SettingsParameters.StateFilePath] path = os.path.join(path, _QUERY_OBJECT_STAGING_FOLDER) self.write({"path": path}) diff --git a/tabpy-server/tabpy_server/management/state.py b/tabpy-server/tabpy_server/management/state.py index b9b0e7ca..8ec2e492 100644 --- a/tabpy-server/tabpy_server/management/state.py +++ b/tabpy-server/tabpy_server/management/state.py @@ -65,7 +65,6 @@ def load_state_from_str(state_string): log_and_raise("State string is empty!", ValueError) - def save_state_to_str(config): ''' Convert from ConfigParser to String diff --git a/tabpy-server/tabpy_server/psws/callbacks.py b/tabpy-server/tabpy_server/psws/callbacks.py index fab69dd2..c8fa327e 100644 --- a/tabpy-server/tabpy_server/psws/callbacks.py +++ b/tabpy-server/tabpy_server/psws/callbacks.py @@ -160,7 +160,8 @@ def on_state_change(settings, tabpy_state, python_service): python_service.manage_request(DeleteObjects([object_name])) - cleanup_endpoint_files(object_name, settings[SettingsParameters.UploadDir]) + cleanup_endpoint_files(object_name, + settings[SettingsParameters.UploadDir]) else: endpoint_info = new_endpoints[object_name] diff --git a/tabpy-tools/tabpy_tools/rest.py b/tabpy-tools/tabpy_tools/rest.py index 0343a102..dfb943b5 100755 --- a/tabpy-tools/tabpy_tools/rest.py +++ b/tabpy-tools/tabpy_tools/rest.py @@ -141,8 +141,10 @@ def PUT(self, url, data, timeout=None): return response.json() def DELETE(self, url, data, timeout=None): - """Issues a DELETE request to the URL with the data specified. Returns an - object that is parsed from the response JSON.""" + ''' + Issues a DELETE request to the URL with the data specified. Returns an + object that is parsed from the response JSON. + ''' if data is not None: data = json.dumps(data) diff --git a/tabpy-tools/tools_tests/test_rest.py b/tabpy-tools/tools_tests/test_rest.py index c4d9cd70..f54d61ce 100644 --- a/tabpy-tools/tools_tests/test_rest.py +++ b/tabpy-tools/tools_tests/test_rest.py @@ -223,7 +223,8 @@ def setUp(self): nw.DELETE.return_value = 'DELETE' self.sc = ServiceClient('endpoint/', network_wrapper=nw) - self.scClientDoesNotEndWithSlash = ServiceClient('endpoint', network_wrapper=nw) + self.scClientDoesNotEndWithSlash =\ + ServiceClient('endpoint', network_wrapper=nw) def test_GET(self): self.assertEqual(self.sc.GET('test'), 'GET') From 9162c79e2b7797152d872bab09f4193ff654c0ea Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 18 Apr 2019 11:15:34 -0700 Subject: [PATCH 08/12] Code review feedback --- .gitignore | 1 - .vscode/settings.json | 10 ++++++ docs/server-config.md | 31 +++++++++++++++++++ tabpy-server/tabpy_server/app/app.py | 8 +++-- .../tabpy_server/handlers/base_handler.py | 3 ++ 5 files changed, 49 insertions(+), 4 deletions(-) create mode 100755 .vscode/settings.json diff --git a/.gitignore b/.gitignore index cfa814e9..a204dcab 100644 --- a/.gitignore +++ b/.gitignore @@ -123,7 +123,6 @@ tabpy-server/tabpy_server/staging # VS Code *.code-workspace -.vscode/ # etc setup.bat diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100755 index 00000000..917bf975 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "git.enabled": true, + "files.exclude": { + "**/__pycache__": true, + "**/.pytest_cache": true + }, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.enabled": true +} \ No newline at end of file diff --git a/docs/server-config.md b/docs/server-config.md index 3c06c851..35947d4e 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -115,3 +115,34 @@ will be generated and displayed in the command line. To delete an account open password file in any text editor and delete the line with the user name. + +## Logging + +Logging for TabPy is implemented with standart Python logger and can be configured +as explained in Python documentation at +[Logging Configuration page](https://docs.python.org/3.6/library/logging.config.html). + +Default config proveded with TabPy is +[`tabpy-server/tabpy_server/common/default.conf`](tabpy-server/tabpy_server/common/default.conf) +and has configuration for console and file loggers. With changing the config +user can modify log level, format of the logges messages and add or remove +loggers. + +### Request Context Logging + +For extended logging (e.g. for auditing purposes) additional logging can be turned +on with setting `TABPY_LOG_DETAILS` configuration file parameter to `true`. + +With the feature on additional information is logged for HTTP requests: caller ip, +URL, client and user information: + + +``` +2019-04-17,15:20:37 [INFO] (evaluation_plane_handler.py:evaluation_plane_handler:86): ::1 calls POST http://localhost:9004/evaluate, Client: Postman for manual testing, Tableau user: ogolovatyi, TabPy user: user1 +function to evaluate=def _user_script(tabpy, _arg1, _arg2): + res = [] + for i in range(len(_arg1)): + res.append(_arg1[i] * _arg2[i]) + return res +``` + diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 94d525af..b5fc005b 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -70,16 +70,18 @@ def run(self): self.tabpy_state, self.python_service) - if self.settings[SettingsParameters.TransferProtocol] == 'http': + protocol = self.settings[SettingsParameters.TransferProtocol] + if protocol == 'http': application.listen(self.settings[SettingsParameters.Port]) - elif self.settings[SettingsParameters.TransferProtocol] == 'https': + elif protocol == 'https': application.listen(self.settings[SettingsParameters.Port], ssl_options={ 'certfile': self.settings[SettingsParameters.CertificateFile], 'keyfile': self.settings[SettingsParameters.KeyFile] }) else: - log_and_raise('Unsupported transfer protocol.', RuntimeError) + log_and_raise(f'Unsupported transfer protocol {protocol}.', + RuntimeError) logger.info('Web service listening on port {}'.format( str(self.settings[SettingsParameters.Port]))) diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index 32ca5dc3..7ea8bc4d 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -91,6 +91,9 @@ def _get_auth_method(self, api_version) -> (bool, str): str Name of authentication method used by client. If empty no authentication required. + + (True, '') as result of this function means authentication + is not needed. ''' if api_version not in self.settings[SettingsParameters.ApiVersions]: logger.critical(f'Unknown API version "{api_version}"') From 567903f54de39d7a786ccf477170053001cf2945 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 18 Apr 2019 11:34:22 -0700 Subject: [PATCH 09/12] Fix markdownlint failure --- docs/server-config.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/server-config.md b/docs/server-config.md index 35947d4e..349f7535 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -137,6 +137,7 @@ With the feature on additional information is logged for HTTP requests: caller i URL, client and user information: + ``` 2019-04-17,15:20:37 [INFO] (evaluation_plane_handler.py:evaluation_plane_handler:86): ::1 calls POST http://localhost:9004/evaluate, Client: Postman for manual testing, Tableau user: ogolovatyi, TabPy user: user1 function to evaluate=def _user_script(tabpy, _arg1, _arg2): @@ -145,4 +146,5 @@ function to evaluate=def _user_script(tabpy, _arg1, _arg2): res.append(_arg1[i] * _arg2[i]) return res ``` + From 0f3abeb8c9d07b514bec83b5c4861a0d9aae6a4e Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 18 Apr 2019 11:37:32 -0700 Subject: [PATCH 10/12] Fix markdownlint failure --- docs/server-config.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/server-config.md b/docs/server-config.md index 349f7535..2777c0c4 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -1,6 +1,7 @@ # TabPy Server Configuration Instructions + - [Configuring HTTP vs HTTPS](#configuring-http-vs-https) @@ -10,8 +11,11 @@ * [Adding an Account](#adding-an-account) * [Updating an Account](#updating-an-account) * [Deleting an Account](#deleting-an-account) +- [Logging](#logging) + * [Request Context Logging](#request-context-logging) + Default settings for TabPy may be viewed in the @@ -137,14 +141,15 @@ With the feature on additional information is logged for HTTP requests: caller i URL, client and user information: - ``` -2019-04-17,15:20:37 [INFO] (evaluation_plane_handler.py:evaluation_plane_handler:86): ::1 calls POST http://localhost:9004/evaluate, Client: Postman for manual testing, Tableau user: ogolovatyi, TabPy user: user1 +2019-04-17,15:20:37 [INFO] (evaluation_plane_handler.py:evaluation_plane_handler:86): + ::1 calls POST http://localhost:9004/evaluate, + Client: Postman for manual testing, Tableau user: ogolovatyi, + TabPy user: user1 function to evaluate=def _user_script(tabpy, _arg1, _arg2): res = [] for i in range(len(_arg1)): res.append(_arg1[i] * _arg2[i]) return res ``` - From a449df75a25bf2e7482d47c0b58a35f140fe3a84 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 18 Apr 2019 17:28:00 -0700 Subject: [PATCH 11/12] Code cleanup --- tabpy-server/tabpy_server/app/app.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index b5fc005b..fe6e4122 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -1,15 +1,10 @@ +from argparse import ArgumentParser import concurrent.futures import configparser -import csv import logging +from logging import config import multiprocessing import os -import tornado - -from argparse import ArgumentParser - -from logging import config - import tabpy_server from tabpy_server import __version__ from tabpy_server.app.ConfigParameters import ConfigParameters @@ -27,6 +22,7 @@ ServiceInfoHandler, StatusHandler, UploadDestinationHandler) from tornado_json.constants import TORNADO_MAJOR +import tornado logger = logging.getLogger(__name__) From f4a7c5c2545d0e4bb70bd02b1c46c56aaeccae21 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 19 Apr 2019 08:52:34 -0700 Subject: [PATCH 12/12] Update doc --- docs/server-config.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/server-config.md b/docs/server-config.md index 2777c0c4..54a5bd91 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -138,14 +138,16 @@ For extended logging (e.g. for auditing purposes) additional logging can be turn on with setting `TABPY_LOG_DETAILS` configuration file parameter to `true`. With the feature on additional information is logged for HTTP requests: caller ip, -URL, client and user information: +URL, client infomation (Tableau Desktop\Server), Tableau user name (for Tableau Server) +and TabPy user name as shown in the example below: ``` 2019-04-17,15:20:37 [INFO] (evaluation_plane_handler.py:evaluation_plane_handler:86): ::1 calls POST http://localhost:9004/evaluate, - Client: Postman for manual testing, Tableau user: ogolovatyi, - TabPy user: user1 + Client: Tableau Server 2019.2, + Tableau user: ogolovatyi, + TabPy user: user1 function to evaluate=def _user_script(tabpy, _arg1, _arg2): res = [] for i in range(len(_arg1)):