diff --git a/.gitignore b/.gitignore index cfa814e9..a204dcab 100644 --- a/.gitignore +++ b/.gitignore @@ -123,7 +123,6 @@ tabpy-server/tabpy_server/staging # VS Code *.code-workspace -.vscode/ # etc setup.bat diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100755 index 00000000..917bf975 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "git.enabled": true, + "files.exclude": { + "**/__pycache__": true, + "**/.pytest_cache": true + }, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.enabled": true +} \ No newline at end of file diff --git a/docs/server-config.md b/docs/server-config.md index 3c06c851..54a5bd91 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -1,6 +1,7 @@ # TabPy Server Configuration Instructions + - [Configuring HTTP vs HTTPS](#configuring-http-vs-https) @@ -10,8 +11,11 @@ * [Adding an Account](#adding-an-account) * [Updating an Account](#updating-an-account) * [Deleting an Account](#deleting-an-account) +- [Logging](#logging) + * [Request Context Logging](#request-context-logging) + Default settings for TabPy may be viewed in the @@ -115,3 +119,39 @@ will be generated and displayed in the command line. To delete an account open password file in any text editor and delete the line with the user name. + +## Logging + +Logging for TabPy is implemented with standart Python logger and can be configured +as explained in Python documentation at +[Logging Configuration page](https://docs.python.org/3.6/library/logging.config.html). + +Default config proveded with TabPy is +[`tabpy-server/tabpy_server/common/default.conf`](tabpy-server/tabpy_server/common/default.conf) +and has configuration for console and file loggers. With changing the config +user can modify log level, format of the logges messages and add or remove +loggers. + +### Request Context Logging + +For extended logging (e.g. for auditing purposes) additional logging can be turned +on with setting `TABPY_LOG_DETAILS` configuration file parameter to `true`. + +With the feature on additional information is logged for HTTP requests: caller ip, +URL, client infomation (Tableau Desktop\Server), Tableau user name (for Tableau Server) +and TabPy user name as shown in the example below: + + +``` +2019-04-17,15:20:37 [INFO] (evaluation_plane_handler.py:evaluation_plane_handler:86): + ::1 calls POST http://localhost:9004/evaluate, + Client: Tableau Server 2019.2, + Tableau user: ogolovatyi, + TabPy user: user1 +function to evaluate=def _user_script(tabpy, _arg1, _arg2): + res = [] + for i in range(len(_arg1)): + res.append(_arg1[i] * _arg2[i]) + return res +``` + diff --git a/tabpy-server/server_tests/test_config.py b/tabpy-server/server_tests/test_config.py index 452da6bd..58a67b53 100644 --- a/tabpy-server/server_tests/test_config.py +++ b/tabpy-server/server_tests/test_config.py @@ -9,7 +9,6 @@ from unittest.mock import patch, call - def assert_raises_runtime_error(message, fn, args={}): try: fn(*args) @@ -96,6 +95,7 @@ def test_config_file_present(self, mock_os, mock_path_exists, self.assertEqual(app.settings['transfer_protocol'], 'http') self.assertTrue('certificate_file' not in app.settings) self.assertTrue('key_file' not in app.settings) + self.assertEqual(app.settings['log_request_context'], False) os.remove(config_file.name) diff --git a/tabpy-server/server_tests/test_evaluation_plane_handler.py b/tabpy-server/server_tests/test_evaluation_plane_handler.py index 5e9a57e2..2f9ad4a6 100755 --- a/tabpy-server/server_tests/test_evaluation_plane_handler.py +++ b/tabpy-server/server_tests/test_evaluation_plane_handler.py @@ -128,7 +128,7 @@ def test_valid_creds_pass(self): def test_null_request(self): response = self.fetch('') - self.assertEqual(599, response.code) + self.assertEqual(404, response.code) def test_script_not_present(self): response = self.fetch( diff --git a/tabpy-server/server_tests/test_service_info_handler.py b/tabpy-server/server_tests/test_service_info_handler.py index 14c8cedf..1f42aa31 100644 --- a/tabpy-server/server_tests/test_service_info_handler.py +++ b/tabpy-server/server_tests/test_service_info_handler.py @@ -2,6 +2,7 @@ import json import os from tabpy_server.app.app import TabPyApp +from tabpy_server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase from unittest.mock import patch @@ -12,7 +13,7 @@ def _create_expected_info_response(settings, tabpy_state): 'description': tabpy_state.get_description(), 'creation_time': tabpy_state.creation_time, 'state_path': settings['state_file_path'], - 'server_version': settings['server_version'], + 'server_version': settings[SettingsParameters.ServerVersion], 'name': tabpy_state.name, 'versions': settings['versions'] } diff --git a/tabpy-server/server_tests/test_validate_credentials.py b/tabpy-server/server_tests/test_validate_credentials.py deleted file mode 100755 index 028c5409..00000000 --- a/tabpy-server/server_tests/test_validate_credentials.py +++ /dev/null @@ -1,198 +0,0 @@ -import base64 -import hashlib -import logging -import pathlib -import os -import unittest -from argparse import Namespace -from tempfile import NamedTemporaryFile - -from tabpy_server.handlers.util import ( - validate_basic_auth_credentials, - handle_basic_authentication, - check_and_validate_basic_auth_credentials) - -from unittest.mock import patch, call - - -class TestValidateBasicAuthCredentials(unittest.TestCase): - def setUp(self): - self.credentials = { - # PBKDF2('user1', 'password', 10000) - 'user1': - ('5961c87343553bf078add1189b4f59238f663eabd23dbc1dc538c5fed' - 'f18a8cc546f4a46500dd7672144595ac4e9610dc9edc66ee1cb7b58cab' - '64ddb662390b3') - } - - def test_given_unknown_username_expect_validation_fails(self): - self.assertFalse(validate_basic_auth_credentials( - 'user2', 'pwd@2', self.credentials)) - - def test_given_wrong_password_expect_validation_fails(self): - self.assertFalse(validate_basic_auth_credentials( - 'user1', 'password#1', self.credentials)) - - def test_given_valid_creds_expect_validation_passes(self): - self.assertTrue(validate_basic_auth_credentials( - 'user1', 'password', self.credentials)) - - def test_given_valid_creds_mixcase_login_expect_validation_passes(self): - self.assertTrue(validate_basic_auth_credentials( - 'UsEr1', 'password', self.credentials)) - - -class TestCheckAndValidateBasicAuthCredentials(unittest.TestCase): - def setUp(self): - self.credentials = { - # PBKDF2('user1', 'password', 10000) - 'user1': - ('5961c87343553bf078add1189b4f59238f663eabd23dbc1dc538c5fed' - 'f18a8cc546f4a46500dd7672144595ac4e9610dc9edc66ee1cb7b58cab' - '64ddb662390b3') - } - - def test_given_no_headers_expect_validation_fails(self): - self.assertFalse( - check_and_validate_basic_auth_credentials({}, self.credentials)) - - def test_given_bad_auth_header_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Some unexpected string' - }, self.credentials)) - - def test_given_bad_encoded_credentials_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic abc' - }, self.credentials)) - - def test_given_malformed_credentials_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format( - base64.b64encode('user1-password'.encode('utf-8')). - decode('utf-8')) - }, self.credentials)) - - def test_given_unknown_username_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format( - base64.b64encode('unknown_user:password'.encode('utf-8'))) - }, self.credentials)) - - def test_given_wrong_pwd_expect_validation_fails(self): - self.assertFalse(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format( - base64.b64encode('user1:p@ssw0rd'.encode('utf-8'))) - }, self.credentials)) - - def test_given_valid_creds_expect_validation_passes(self): - b64_username_pwd = base64.b64encode( - 'user1:password'.encode('utf-8')).decode('utf-8') - self.assertTrue(check_and_validate_basic_auth_credentials( - { - 'Authorization': 'Basic {}'.format(b64_username_pwd) - }, self.credentials)) - - -class TestHandleAuthentication(unittest.TestCase): - def setUp(self): - self.credentials = { - # PBKDF2('user1', 'password', 10000) - 'user1': - ('5961c87343553bf078add1189b4f59238f663eabd23dbc1dc538c5fed' - 'f18a8cc546f4a46500dd7672144595ac4e9610dc9edc66ee1cb7b58cab' - '64ddb662390b3') - } - - self.settings = { - 'versions': - { - 'v0.1a': - { - 'features': {} - }, - 'v0.2beta': - { - 'features': - { - 'authentication': - { - 'required': True, - } - } - }, - 'v0.3gamma': - { - 'features': - { - 'authentication': - { - 'required': True, - 'methods': - { - 'unknown-auth': {} - } - } - } - }, - 'v0.4yota': {}, - 'v1': - { - 'features': - { - 'authentication': - { - 'required': True, - 'methods': - { - 'basic-auth': {} - } - } - } - } - } - } - - def test_given_no_api_version_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, '', self.settings, self.credentials)) - - def test_given_unknown_api_version_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v0.314p', self.settings, self.credentials)) - - def test_given_auth_is_not_configured_expect_success(self): - self.assertTrue(handle_basic_authentication( - {}, 'v0.1a', self.settings, self.credentials)) - - def test_given_auth_method_not_provided_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v0.2beta', self.settings, self.credentials)) - - def test_given_auth_method_is_unknown_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v0.3gamma', self.settings, self.credentials)) - - def test_given_features_not_configured_expect_success(self): - self.assertTrue(handle_basic_authentication( - {}, 'v0.4yota', self.settings, self.credentials)) - - def test_given_headers_not_provided_expect_failure(self): - self.assertFalse(handle_basic_authentication( - {}, 'v1', self.settings, self.credentials)) - - def test_given_valid_creds_expect_success(self): - b64_username_pwd = base64.b64encode( - 'user1:password'.encode('utf-8')).decode('utf-8') - self.assertTrue(handle_basic_authentication( - { - 'Authorization': 'Basic {}'.format(b64_username_pwd) - }, - 'v1', - self.settings, - self.credentials)) diff --git a/tabpy-server/tabpy_server/app/ConfigParameters.py b/tabpy-server/tabpy_server/app/ConfigParameters.py index d333dd16..0af2c5af 100644 --- a/tabpy-server/tabpy_server/app/ConfigParameters.py +++ b/tabpy-server/tabpy_server/app/ConfigParameters.py @@ -10,3 +10,5 @@ class ConfigParameters: TABPY_CERTIFICATE_FILE = 'TABPY_CERTIFICATE_FILE' TABPY_KEY_FILE = 'TABPY_KEY_FILE' TABPY_PWD_FILE = 'TABPY_PWD_FILE' + TABPY_LOG_DETAILS = 'TABPY_LOG_DETAILS' + TABPY_STATIC_PATH = 'TABPY_STATIC_PATH' diff --git a/tabpy-server/tabpy_server/app/SettingsParameters.py b/tabpy-server/tabpy_server/app/SettingsParameters.py new file mode 100755 index 00000000..b070a2b4 --- /dev/null +++ b/tabpy-server/tabpy_server/app/SettingsParameters.py @@ -0,0 +1,14 @@ +class SettingsParameters: + ''' + Application (TabPyApp) settings names + ''' + TransferProtocol = 'transfer_protocol' + Port = 'port' + ServerVersion = 'server_version' + UploadDir = 'upload_dir' + CertificateFile = 'certificate_file' + KeyFile = 'key_file' + StateFilePath = 'state_file_path' + ApiVersions = 'versions' + LogRequestContext = 'log_request_context' + StaticPath = 'static_path' diff --git a/tabpy-server/tabpy_server/app/app.py b/tabpy-server/tabpy_server/app/app.py index 5539a211..fe6e4122 100644 --- a/tabpy-server/tabpy_server/app/app.py +++ b/tabpy-server/tabpy_server/app/app.py @@ -1,18 +1,14 @@ +from argparse import ArgumentParser import concurrent.futures import configparser -import csv import logging +from logging import config import multiprocessing import os -import tornado - -from argparse import ArgumentParser - -from logging import config - import tabpy_server from tabpy_server import __version__ from tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.app.util import ( log_and_raise, parse_pwd_file) @@ -25,8 +21,8 @@ EvaluationPlaneHandler, QueryPlaneHandler, ServiceInfoHandler, StatusHandler, UploadDestinationHandler) - from tornado_json.constants import TORNADO_MAJOR +import tornado logger = logging.getLogger(__name__) @@ -70,19 +66,21 @@ def run(self): self.tabpy_state, self.python_service) - if self.settings['transfer_protocol'] == 'http': - application.listen(self.settings['port']) - elif self.settings['transfer_protocol'] == 'https': - application.listen(self.settings['port'], + protocol = self.settings[SettingsParameters.TransferProtocol] + if protocol == 'http': + application.listen(self.settings[SettingsParameters.Port]) + elif protocol == 'https': + application.listen(self.settings[SettingsParameters.Port], ssl_options={ - 'certfile': self.settings['certificate_file'], - 'keyfile': self.settings['key_file'] + 'certfile': self.settings[SettingsParameters.CertificateFile], + 'keyfile': self.settings[SettingsParameters.KeyFile] }) else: - log_and_raise('Unsupported transfer protocol.', RuntimeError) + log_and_raise(f'Unsupported transfer protocol {protocol}.', + RuntimeError) logger.info('Web service listening on port {}'.format( - str(self.settings['port']))) + str(self.settings[SettingsParameters.Port]))) tornado.ioloop.IOLoop.instance().start() def _create_tornado_web_app(self): @@ -117,9 +115,8 @@ def _create_tornado_web_app(self): UploadDestinationHandler, dict(app=self)), (self.subdirectory + r'/(.*)', tornado.web.StaticFileHandler, - dict(path=self.settings['static_path'], - default_filename="index.html", - app=self)), + dict(path=self.settings[SettingsParameters.StaticPath], + default_filename="index.html")), ], debug=False, **self.settings) return application @@ -184,36 +181,41 @@ def set_parameter(settings_key, elif default_val is not None: self.settings[settings_key] = default_val - set_parameter('port', ConfigParameters.TABPY_PORT, + set_parameter(SettingsParameters.Port, ConfigParameters.TABPY_PORT, default_val=9004, check_env_var=True) - set_parameter('server_version', None, default_val=__version__) + set_parameter(SettingsParameters.ServerVersion, None, + default_val=__version__) - set_parameter('upload_dir', ConfigParameters.TABPY_QUERY_OBJECT_PATH, + set_parameter(SettingsParameters.UploadDir, + ConfigParameters.TABPY_QUERY_OBJECT_PATH, default_val='/tmp/query_objects', check_env_var=True) - if not os.path.exists(self.settings['upload_dir']): - os.makedirs(self.settings['upload_dir']) + if not os.path.exists(self.settings[SettingsParameters.UploadDir]): + os.makedirs(self.settings[SettingsParameters.UploadDir]) # set and validate transfer protocol - set_parameter('transfer_protocol', + set_parameter(SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL, default_val='http') - self.settings['transfer_protocol'] =\ - self.settings['transfer_protocol'].lower() + self.settings[SettingsParameters.TransferProtocol] =\ + self.settings[SettingsParameters.TransferProtocol].lower() - set_parameter('certificate_file', + set_parameter(SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE) - set_parameter('key_file', ConfigParameters.TABPY_KEY_FILE) + set_parameter(SettingsParameters.KeyFile, + ConfigParameters.TABPY_KEY_FILE) self._validate_transfer_protocol_settings() # if state.ini does not exist try and create it - remove # last dependence on batch/shell script - set_parameter('state_file_path', ConfigParameters.TABPY_STATE_PATH, + set_parameter(SettingsParameters.StateFilePath, + ConfigParameters.TABPY_STATE_PATH, default_val='./tabpy-server/tabpy_server', check_env_var=True) - self.settings['state_file_path'] = os.path.realpath( + self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( - os.path.expanduser(self.settings['state_file_path']))) - state_file_path = self.settings['state_file_path'] + os.path.expanduser( + self.settings[SettingsParameters.StateFilePath]))) + state_file_path = self.settings[SettingsParameters.StateFilePath] logger.info("Loading state from state file %s" % os.path.join(state_file_path, "state.ini")) tabpy_state = _get_state_from_file(state_file_path) @@ -223,8 +225,14 @@ def set_parameter(settings_key, self.python_service = PythonServiceHandler(PythonService()) self.settings['compress_response'] = True if TORNADO_MAJOR >= 4\ else "gzip" - self.settings['static_path'] = os.path.join( - os.path.dirname(__file__), "static") + + set_parameter(SettingsParameters.StaticPath, + ConfigParameters.TABPY_STATIC_PATH, + default_val='./') + self.settings[SettingsParameters.StaticPath] =\ + os.path.abspath(self.settings[SettingsParameters.StaticPath]) + logger.debug(f'Static pages folder set to ' + '"{self.settings[SettingsParameters.StaticPath]}"') # Set subdirectory from config if applicable if tabpy_state.has_option("Service Info", "Subdirectory"): @@ -245,14 +253,25 @@ def set_parameter(settings_key, "Authentication is not enabled") features = self._get_features() - self.settings['versions'] = {'v1': {'features': features}} + self.settings[SettingsParameters.ApiVersions] =\ + {'v1': {'features': features}} + + set_parameter(SettingsParameters.LogRequestContext, + ConfigParameters.TABPY_LOG_DETAILS, + default_val='false') + self.settings[SettingsParameters.LogRequestContext] = ( + self.settings[SettingsParameters.LogRequestContext].lower() != + 'false') + logger.info('Call context logging is {}'.format( + 'enabled' if self.settings[SettingsParameters.LogRequestContext] + else 'disabled')) def _validate_transfer_protocol_settings(self): - if 'transfer_protocol' not in self.settings: + if SettingsParameters.TransferProtocol not in self.settings: log_and_raise( 'Missing transfer protocol information.', RuntimeError) - protocol = self.settings['transfer_protocol'] + protocol = self.settings[SettingsParameters.TransferProtocol] if protocol == 'http': return @@ -261,16 +280,17 @@ def _validate_transfer_protocol_settings(self): log_and_raise('Unsupported transfer protocol: {}.'.format( protocol), RuntimeError) - self._validate_cert_key_state('The parameter(s) {} must be set.', - 'certificate_file' in self.settings, - 'key_file' in self.settings) - cert = self.settings['certificate_file'] + self._validate_cert_key_state( + 'The parameter(s) {} must be set.', + SettingsParameters.CertificateFile in self.settings, + SettingsParameters.KeyFile in self.settings) + cert = self.settings[SettingsParameters.CertificateFile] self._validate_cert_key_state( 'The parameter(s) {} must point to ' 'an existing file.', os.path.isfile(cert), - os.path.isfile(self.settings['key_file'])) + os.path.isfile(self.settings[SettingsParameters.KeyFile])) tabpy_server.app.util.validate_cert(cert) @staticmethod diff --git a/tabpy-server/tabpy_server/common/default.conf b/tabpy-server/tabpy_server/common/default.conf index de0887b6..6eef5265 100755 --- a/tabpy-server/tabpy_server/common/default.conf +++ b/tabpy-server/tabpy_server/common/default.conf @@ -3,6 +3,9 @@ TABPY_QUERY_OBJECT_PATH = /tmp/query_objects TABPY_PORT = 9004 TABPY_STATE_PATH = ./tabpy-server/tabpy_server +# Where static pages live +TABPY_STATIC_PATH = ./tabpy-server/tabpy_server/static + # For how to configure TabPy authentication read # docs/server-config.md. # TABPY_PWD_FILE = /path/to/password/file.txt @@ -13,6 +16,10 @@ TABPY_STATE_PATH = ./tabpy-server/tabpy_server # TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt # TABPY_KEY_FILE = path/to/key/file.key +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + [loggers] keys=root diff --git a/tabpy-server/tabpy_server/common/endpoint_file_mgr.py b/tabpy-server/tabpy_server/common/endpoint_file_mgr.py index d46d1dfd..d1b79beb 100644 --- a/tabpy-server/tabpy_server/common/endpoint_file_mgr.py +++ b/tabpy-server/tabpy_server/common/endpoint_file_mgr.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) -_name_checker = _compile('^[a-zA-Z0-9-_\ ]+$') +_name_checker = _compile(r'^[a-zA-Z0-9-_\s]+$') def _check_endpoint_name(name): diff --git a/tabpy-server/tabpy_server/handlers/__init__.py b/tabpy-server/tabpy_server/handlers/__init__.py index 59af73b7..7dc480b4 100644 --- a/tabpy-server/tabpy_server/handlers/__init__.py +++ b/tabpy-server/tabpy_server/handlers/__init__.py @@ -11,4 +11,3 @@ from tabpy_server.handlers.status_handler import StatusHandler from tabpy_server.handlers.upload_destination_handler\ import UploadDestinationHandler -from tabpy_server.handlers.util import handle_basic_authentication diff --git a/tabpy-server/tabpy_server/handlers/base_handler.py b/tabpy-server/tabpy_server/handlers/base_handler.py index 621189e1..7ea8bc4d 100644 --- a/tabpy-server/tabpy_server/handlers/base_handler.py +++ b/tabpy-server/tabpy_server/handlers/base_handler.py @@ -1,9 +1,12 @@ +import base64 +import binascii import concurrent import tornado.web import json import logging -from tabpy_server.handlers.util import handle_basic_authentication +from tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy_server.handlers.util import hash_password logger = logging.getLogger(__name__) STAGING_THREAD = concurrent.futures.ThreadPoolExecutor(max_workers=3) @@ -16,9 +19,12 @@ def initialize(self, app): self.tabpy_state = app.tabpy_state # set content type to application/json self.set_header("Content-Type", "application/json") - self.port = self.settings['port'] + self.port = self.settings[SettingsParameters.Port] self.python_service = app.python_service self.credentials = app.credentials + self.log_request_context =\ + app.settings[SettingsParameters.LogRequestContext] + self.username = None def error_out(self, code, log_message, info=None): self.set_status(code) @@ -47,23 +53,230 @@ def _add_CORS_header(self): origin = self.tabpy_state.get_access_control_allow_origin() if len(origin) > 0: self.set_header("Access-Control-Allow-Origin", origin) - logger.debug("Access-Control-Allow-Origin:{}".format(origin)) + logger.debug(self.append_request_context( + "Access-Control-Allow-Origin:{}".format(origin))) headers = self.tabpy_state.get_access_control_allow_headers() if len(headers) > 0: self.set_header("Access-Control-Allow-Headers", headers) - logger.debug("Access-Control-Allow-Headers:{}".format(headers)) + logger.debug(self.append_request_context( + "Access-Control-Allow-Headers:{}".format(headers))) methods = self.tabpy_state.get_access_control_allow_methods() if len(methods) > 0: self.set_header("Access-Control-Allow-Methods", methods) - logger.debug("Access-Control-Allow-Methods:{}".format(methods)) + logger.debug(self.append_request_context( + "Access-Control-Allow-Methods:{}".format(methods))) def _sanitize_request_data(self, data, keys=KEYS_TO_SANITIZE): """Remove keys so that we can log safely""" for key in keys: data.pop(key, None) + def _get_auth_method(self, api_version) -> (bool, str): + ''' + Finds authentication method if provided. + + Parameters + ---------- + api_version : str + API version for authentication. + + Returns + ------- + bool + True if known authentication method is found. + False otherwise. + + str + Name of authentication method used by client. + If empty no authentication required. + + (True, '') as result of this function means authentication + is not needed. + ''' + if api_version not in self.settings[SettingsParameters.ApiVersions]: + logger.critical(f'Unknown API version "{api_version}"') + return False, '' + + version_settings =\ + self.settings[SettingsParameters.ApiVersions][api_version] + if 'features' not in version_settings: + logger.info(f'No features configured for API "{api_version}"') + return True, '' + + features = version_settings['features'] + if 'authentication' not in features or\ + not features['authentication']['required']: + logger.info( + f'Authentication is not a required feature for API ' + '"{api_version}"') + return True, '' + + auth_feature = features['authentication'] + if 'methods' not in auth_feature: + logger.critical( + f'Authentication method is not configured for API ' + '"{api_version}"') + + methods = auth_feature['methods'] + if 'basic-auth' in auth_feature['methods']: + return True, 'basic-auth' + # Add new methods here... + + # No known methods were found + logger.critical( + f'Unknown authentication method(s) "{methods}" are configured ' + 'for API "{api_version}"') + return False, '' + + def _get_basic_auth_credentials(self) -> bool: + ''' + Find credentials for basic access authentication method. Credentials if + found stored in self.username and self.password. + + Returns + ------- + bool + True if valid credentials were found. + False otherwise. + ''' + logger.debug('Checking request headers for authentication data') + if 'Authorization' not in self.request.headers: + logger.info('Authorization header not found') + return False + + auth_header = self.request.headers['Authorization'] + auth_header_list = auth_header.split(' ') + if len(auth_header_list) != 2 or\ + auth_header_list[0] != 'Basic': + logger.error(f'Unknown authentication method "{auth_header}"') + return False + + try: + cred = base64.b64decode(auth_header_list[1]).decode('utf-8') + except (binascii.Error, UnicodeDecodeError) as ex: + logger.critical(f'Cannot decode credentials: {str(ex)}') + return False + + login_pwd = cred.split(':') + if len(login_pwd) != 2: + logger.error('Invalid string in encoded credentials') + return False + + self.username = login_pwd[0] + self.password = login_pwd[1] + return True + + def _get_credentials(self, method) -> bool: + ''' + Find credentials for specified authentication method. Credentials if + found stored in self.username and self.password. + + Parameters + ---------- + method: str + Authentication method name. + + Returns + ------- + bool + True if valid credentials were found. + False otherwise. + ''' + if method == 'basic-auth': + return self._get_basic_auth_credentials() + # Add new methods here... + + # No known methods were found + logger.critical( + f'Unknown authentication method(s) "{method}" are configured ' + 'for API "{api_version}"') + return False + + def _validate_basic_auth_credentials(self) -> bool: + ''' + Validates username:pwd if they are the same as + stored credentials. + + Returns + ------- + bool + True if credentials has key login and + credentials[login] equal SHA3(pwd), False + otherwise. + ''' + login = self.username.lower() + logger.debug(f'Validating credentials for user name "{login}"') + if login not in self.credentials: + logger.error(f'User name "{self.username}" not found') + return False + + hashed_pwd = hash_password(self.username, self.password) + if self.credentials[login].lower() != hashed_pwd.lower(): + logger.error(f'Wrong password for user name "{self.username}"') + return False + + return True + + def _validate_credentials(self, method) -> bool: + ''' + Validates credentials according to specified methods if they + are what expected. + + Parameters + ---------- + method: str + Authentication method name. + + Returns + ------- + bool + True if credentials are valid. + False otherwise. + ''' + if method == 'basic-auth': + return self._validate_basic_auth_credentials() + # Add new methods here... + + # No known methods were found + logger.critical( + f'Unknown authentication method(s) "{method}" are configured ' + 'for API "{api_version}"') + return False + + def handle_authentication(self, api_version) -> bool: + ''' + If authentication feature is configured checks provided + credentials. + + Parameters + ---------- + api_version : str + API version for authentication. + + Returns + ------- + bool + True if authentication is not required. + True if authentication is required and valid + credentials provided. + False otherwise. + ''' + logger.debug('Handling authentication') + found, method = self._get_auth_method(api_version) + if not found: + return False + + if method == '': + # Do not validate credentials + return True + + if not self._get_credentials(method): + return False + + return self._validate_credentials(method) + def should_fail_with_not_authorized(self): ''' Checks if authentication is required: @@ -77,18 +290,16 @@ def should_fail_with_not_authorized(self): required and validation for credentials passes. True if validation for credentials failed. ''' - logger.debug('Checking if need to handle authentication') - return not handle_basic_authentication( - self.request.headers, - "v1", - self.settings, - self.credentials) + logger.debug(self.append_request_context( + 'Checking if need to handle authentication')) + return not self.handle_authentication("v1") def fail_with_not_authorized(self): ''' Prepares server 401 response. ''' - logger.error('Failing with 401 for unauthorized request') + logger.error(self.append_request_context( + 'Failing with 401 for unauthorized request')) self.set_status(401) self.set_header('WWW-Authenticate', 'Basic realm="{}"'.format(self.tabpy_state.name)) @@ -96,3 +307,23 @@ def fail_with_not_authorized(self): 401, info="Unauthorized request.", log_message="Invalid credentials provided.") + + def append_request_context(self, msg) -> str: + ''' + Adds request context (caller info) to logged messages. + ''' + context = '' + if self.log_request_context: + # log request details + context = (f'{self.request.remote_ip} calls ' + '{self.request.method} {self.request.full_url()}') + if 'TabPy-Client' in self.request.headers: + context += f', Client: {self.request.headers["TabPy-Client"]}' + if 'TabPy-User' in self.request.headers: + context +=\ + f', Tableau user: {self.request.headers["TabPy-User"]}' + if self.username is not None and self.username != '': + context += f', TabPy user: {self.username}' + context += '\n' + + return context + msg diff --git a/tabpy-server/tabpy_server/handlers/endpoint_handler.py b/tabpy-server/tabpy_server/handlers/endpoint_handler.py index 5a2e0fd3..f5a69a9d 100644 --- a/tabpy-server/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoint_handler.py @@ -27,11 +27,13 @@ def initialize(self, app): super(EndpointHandler, self).initialize(app) def get(self, endpoint_name): - logger.debug('Processing GET for /endpoints/{}'.format(endpoint_name)) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing GET for /endpoints/{endpoint_name}')) + self._add_CORS_header() if not endpoint_name: self.write(json.dumps(self.tabpy_state.get_endpoints())) @@ -46,11 +48,13 @@ def get(self, endpoint_name): @tornado.web.asynchronous @gen.coroutine def put(self, name): - logger.debug('Processing PUT for /endpoints/{}'.format(name)) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing PUT for /endpoints/{name}')) + try: if not self.request.body: self.error_out(400, "Input body cannot be empty") @@ -76,7 +80,8 @@ def put(self, name): return new_version = int(endpoints[name]['version']) + 1 - logger.info('Endpoint info: %s' % request_data) + logger.info(self.append_request_context( + 'Endpoint info: %s' % request_data)) err_msg = yield self._add_or_update_endpoint( 'update', name, new_version, request_data) if err_msg: @@ -94,11 +99,13 @@ def put(self, name): @tornado.web.asynchronous @gen.coroutine def delete(self, name): - logger.debug('Processing DELETE for /endpoints/{}'.format(name)) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing DELETE for /endpoints/{}'.format(name))) + try: endpoints = self.tabpy_state.get_endpoints(name) if len(endpoints) == 0: diff --git a/tabpy-server/tabpy_server/handlers/endpoints_handler.py b/tabpy-server/tabpy_server/handlers/endpoints_handler.py index b18c8a8b..88557cea 100644 --- a/tabpy-server/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy-server/tabpy_server/handlers/endpoints_handler.py @@ -32,11 +32,13 @@ def get(self): @tornado.web.asynchronous @gen.coroutine def post(self): - logger.debug('Processing POST for /endpoints') if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing POST for /endpoints')) + try: if not self.request.body: self.error_out(400, "Input body cannot be empty") @@ -68,13 +70,15 @@ def post(self): self.finish() return - logger.debug("Adding endpoint '{}'".format(name)) + logger.debug(self.append_request_context( + "Adding endpoint '{}'".format(name))) err_msg = yield self._add_or_update_endpoint('add', name, 1, request_data) if err_msg: self.error_out(400, err_msg) else: - logger.debug("Endpoint {} successfully added".format(name)) + logger.debug(self.append_request_context( + "Endpoint {} successfully added".format(name))) self.set_status(201) self.write(self.tabpy_state.get_endpoints(name)) self.finish() diff --git a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py index a48fb4b2..222d32b0 100644 --- a/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/evaluation_plane_handler.py @@ -39,11 +39,13 @@ def initialize(self, executor, app): @tornado.web.asynchronous @gen.coroutine def post(self): - logger.debug('Processing POST for /evaluate') if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + 'Processing POST for /evaluate')) + self._add_CORS_header() try: body = json.loads(self.request.body.decode('utf-8')) @@ -80,8 +82,8 @@ def post(self): for u in user_code.splitlines(): function_to_evaluate += ' ' + u + '\n' - logger.info( - "function to evaluate=%s" % function_to_evaluate) + logger.info(self.append_request_context( + f'function to evaluate={function_to_evaluate}')) result = yield self.call_subprocess(function_to_evaluate, arguments) diff --git a/tabpy-server/tabpy_server/handlers/management_handler.py b/tabpy-server/tabpy_server/handlers/management_handler.py index 8615c410..ccd914d3 100644 --- a/tabpy-server/tabpy_server/handlers/management_handler.py +++ b/tabpy-server/tabpy_server/handlers/management_handler.py @@ -8,6 +8,7 @@ from tornado import gen +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import MainHandler from tabpy_server.handlers.base_handler import STAGING_THREAD from tabpy_server.management.state import get_query_object_path @@ -42,7 +43,7 @@ def copy_from_local(localpath, remotepath, is_dir=False): class ManagementHandler(MainHandler): def initialize(self, app): super(ManagementHandler, self).initialize(app) - self.port = self.settings['port'] + self.port = self.settings[SettingsParameters.Port] def _get_protocol(self): return 'http://' @@ -94,7 +95,7 @@ def _add_or_update_endpoint(self, action, name, version, request_data): src_path = (request_data['src_path'] if 'src_path' in request_data else None) target_path = get_query_object_path( - self.settings['state_file_path'], name, version) + self.settings[SettingsParameters.StateFilePath], name, version) _path_checker = _compile('^[\\a-zA-Z0-9-_\\s/]+$') # copy from staging if src_path: diff --git a/tabpy-server/tabpy_server/handlers/query_plane_handler.py b/tabpy-server/tabpy_server/handlers/query_plane_handler.py index e0ad556f..a28a63fd 100644 --- a/tabpy-server/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy-server/tabpy_server/handlers/query_plane_handler.py @@ -71,18 +71,21 @@ def _query(self, po_name, data, uid, qry): 'utf-8')).hexdigest()) return (QuerySuccessful, response.for_json(), gls_time) else: - logger.error("Failed query, response: {}".format(response)) + logger.error(self.append_request_context( + f'Failed query, response: {response}')) return (type(response), response.for_json(), gls_time) # handle HTTP Options requests to support CORS # don't check API key (client does not send or receive data for OPTIONS, # it just allows the client to subsequently make a POST request) def options(self, pred_name): - logger.debug('Processing OPTIONS for /query/{}'.format(pred_name)) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing OPTIONS for /query/{pred_name}')) + # add CORS headers if TabPy has a cors_origin specified self._add_CORS_header() self.write({}) @@ -150,8 +153,8 @@ def _process_query(self, endpoint_name, start): return if po_name != endpoint_name: - logger.info( - "Querying actual model: po_name={}".format(po_name)) + logger.info(self.append_request_context( + f'Querying actual model: po_name={po_name}')) uid = _get_uuid() @@ -198,30 +201,32 @@ def _get_actual_model(self, endpoint_name): @tornado.web.asynchronous def get(self, endpoint_name): - logger.debug('Processing GET for /query/{}'.format(endpoint_name)) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing GET for /query/{endpoint_name}')) + start = time.time() if sys.version_info > (3, 0): endpoint_name = urllib.parse.unquote(endpoint_name) else: endpoint_name = urllib.unquote(endpoint_name) - logger.debug("GET /query/{}".format(endpoint_name)) self._process_query(endpoint_name, start) @tornado.web.asynchronous def post(self, endpoint_name): - logger.debug('Processing POST for /query/{}'.format(endpoint_name)) if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return + logger.debug(self.append_request_context( + f'Processing POST for /query/{endpoint_name}')) + start = time.time() if sys.version_info > (3, 0): endpoint_name = urllib.parse.unquote(endpoint_name) else: endpoint_name = urllib.unquote(endpoint_name) - logger.debug("POST /query/{}".format(endpoint_name)) self._process_query(endpoint_name, start) diff --git a/tabpy-server/tabpy_server/handlers/service_info_handler.py b/tabpy-server/tabpy_server/handlers/service_info_handler.py index 24da7510..74fbc8e8 100644 --- a/tabpy-server/tabpy_server/handlers/service_info_handler.py +++ b/tabpy-server/tabpy_server/handlers/service_info_handler.py @@ -1,7 +1,12 @@ import json +import logging +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import ManagementHandler +logger = logging.getLogger(__name__) + + class ServiceInfoHandler(ManagementHandler): def initialize(self, app): super(ServiceInfoHandler, self).initialize(app) @@ -10,12 +15,15 @@ def get(self): # do not check for authentication - this method # is the only way for client to collect info about # supported API versions and required features + logger.debug(self.append_request_context( + 'Processing GET for /info')) self._add_CORS_header() info = {} info['description'] = self.tabpy_state.get_description() info['creation_time'] = self.tabpy_state.creation_time - info['state_path'] = self.settings['state_file_path'] - info['server_version'] = self.settings['server_version'] + info['state_path'] = self.settings[SettingsParameters.StateFilePath] + info['server_version'] =\ + self.settings[SettingsParameters.ServerVersion] info['name'] = self.tabpy_state.name - info['versions'] = self.settings['versions'] + info['versions'] = self.settings[SettingsParameters.ApiVersions] self.write(json.dumps(info)) diff --git a/tabpy-server/tabpy_server/handlers/status_handler.py b/tabpy-server/tabpy_server/handlers/status_handler.py index eccd67fe..4297f98f 100644 --- a/tabpy-server/tabpy_server/handlers/status_handler.py +++ b/tabpy-server/tabpy_server/handlers/status_handler.py @@ -17,7 +17,8 @@ def get(self): self._add_CORS_header() - logger.debug("Obtaining service status") + logger.debug(self.append_request_context( + "Obtaining service status")) status_dict = {} for k, v in self.python_service.ps.query_objects.items(): status_dict[k] = { @@ -26,7 +27,8 @@ def get(self): 'status': v['status'], 'last_error': v['last_error']} - logger.debug("Found models: {}".format(status_dict)) + logger.debug(self.append_request_context( + f'Found models: {status_dict}')) self.write(json.dumps(status_dict)) self.finish() return diff --git a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py index c7dff1b0..33a559f1 100644 --- a/tabpy-server/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy-server/tabpy_server/handlers/upload_destination_handler.py @@ -1,4 +1,5 @@ import logging +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.handlers import ManagementHandler import os @@ -13,12 +14,13 @@ def initialize(self, app): super(UploadDestinationHandler, self).initialize(app) def get(self): - logger.debug( - 'Processing GET for /configurations/endpoint_upload_destination') if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return - path = self.settings['state_file_path'] + logger.debug(self.append_request_context( + 'Processing GET for /configurations/endpoint_upload_destination')) + + path = self.settings[SettingsParameters.StateFilePath] path = os.path.join(path, _QUERY_OBJECT_STAGING_FOLDER) self.write({"path": path}) diff --git a/tabpy-server/tabpy_server/handlers/util.py b/tabpy-server/tabpy_server/handlers/util.py index 58ffde19..99617d87 100755 --- a/tabpy-server/tabpy_server/handlers/util.py +++ b/tabpy-server/tabpy_server/handlers/util.py @@ -2,6 +2,7 @@ import binascii from hashlib import pbkdf2_hmac import logging +from tabpy_server.app.SettingsParameters import SettingsParameters logger = logging.getLogger(__name__) @@ -35,151 +36,3 @@ def hash_password(username, pwd): salt=salt.encode(), iterations=10000) return binascii.hexlify(hash).decode() - - -def validate_basic_auth_credentials(username, pwd, credentials): - ''' - Validates username:pwd if they are the same as - stored credentials. - - Parameters - ---------- - username : str - User name (login). - - pwd : str - Password in plain text. The function will hash - the password with SHA3 to compare with hashed - passwords in stored credentials. - - credentials: dict - Dictionary of stored credentials where keys are - user names and values are SHA3-hashed passwords. - - Returns - ------- - bool - True if credentials has key login and - credentials[login] equal SHA3(pwd), False - otherwise. - ''' - login = username.lower() - logger.debug('Validating credentials for user name "{}"'.format(login)) - if login not in credentials: - logger.error('User name "{}" not found'.format(username)) - return False - - hashed_pwd = hash_password(username, pwd) - if credentials[login].lower() != hashed_pwd.lower(): - logger.error('Wrong password for user name "{}"'.format(username)) - return False - - return True - - -def check_and_validate_basic_auth_credentials(headers, credentials): - ''' - Checks if credentials are provided in headers and - if they are valid. - - Parameters - ---------- - headers - HTTP request headers. - - credentials: dict - Dictionary of stored credentials where keys are - user names and values are SHA3-hashed passwords. - - Returns - ------- - bool - True if credentials are present in headers and - they are valid. - ''' - logger.debug('Checking request headers for authentication data') - if 'Authorization' not in headers: - logger.info('Authorization header not found') - return False - - auth_header = headers['Authorization'] - auth_header_list = headers['Authorization'].split(' ') - if len(auth_header_list) != 2 or\ - auth_header_list[0] != 'Basic': - logger.error('Unknown authentication method "{}"'.format(auth_header)) - return False - - try: - cred = base64.b64decode(auth_header_list[1]).decode('utf-8') - except (binascii.Error, UnicodeDecodeError) as ex: - logger.critical('Cannot decode credentials: {}'.format(str(ex))) - return False - - login_pwd = cred.split(':') - if len(login_pwd) != 2: - logger.error('Invalid string in encoded credentials') - return False - - return validate_basic_auth_credentials(login_pwd[0], - login_pwd[1], - credentials) - - -def handle_basic_authentication(headers, api_version, settings, credentials): - ''' - Checks if credentials need to be validated and they are - validates them. - - Parameters - ---------- - headers - HTTP request headers. - - api_version : str - API version for authentication. - - settings : dict - Application settings (TabPyApp.settings). - - credentials: dict - Dictionary of stored credentials where keys are - user names and values are SHA3-hashed passwords. - - Returns - ------- - bool - If for the specified API version authentication is - not turned on returns True. - Otherwise checks what authentication type is used - and if it is supported type validates provided - credentials. - If authentication type is supported and credentials - are valid returns True, otherwise False. - ''' - logger.debug('Handling authentication for request') - if api_version not in settings['versions']: - logger.critical('Unknown API version "{}"'.format(api_version)) - return False - - version_settings = settings['versions'][api_version] - if 'features' not in version_settings: - logger.info('No features configured for API {}'.format(api_version)) - return True - - features = version_settings['features'] - if 'authentication' not in features or\ - not features['authentication']['required']: - logger.info( - 'Authentication is not a required feature for API ' - '{}'.format(api_version)) - return True - - auth_feature = features['authentication'] - if 'methods' not in auth_feature or\ - 'basic-auth' not in auth_feature['methods']: - logger.critical( - 'Basic authentication access method is not configured ' - 'for API {}'.format(api_version)) - return False - - return check_and_validate_basic_auth_credentials(headers, credentials) diff --git a/tabpy-server/tabpy_server/management/state.py b/tabpy-server/tabpy_server/management/state.py index b9b0e7ca..8ec2e492 100644 --- a/tabpy-server/tabpy_server/management/state.py +++ b/tabpy-server/tabpy_server/management/state.py @@ -65,7 +65,6 @@ def load_state_from_str(state_string): log_and_raise("State string is empty!", ValueError) - def save_state_to_str(config): ''' Convert from ConfigParser to String diff --git a/tabpy-server/tabpy_server/management/util.py b/tabpy-server/tabpy_server/management/util.py index bdc175d5..9a05fd39 100644 --- a/tabpy-server/tabpy_server/management/util.py +++ b/tabpy-server/tabpy_server/management/util.py @@ -6,6 +6,7 @@ from configparser import ConfigParser as _ConfigParser from datetime import datetime, timedelta, tzinfo from tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.app.util import log_and_raise from time import mktime @@ -13,8 +14,8 @@ def write_state_config(state, settings): - if 'state_file_path' in settings: - state_path = settings['state_file_path'] + if SettingsParameters.StateFilePath in settings: + state_path = settings[SettingsParameters.StateFilePath] else: log_and_raise( '{} is not set'.format( diff --git a/tabpy-server/tabpy_server/psws/callbacks.py b/tabpy-server/tabpy_server/psws/callbacks.py index 2f4f1610..c8fa327e 100644 --- a/tabpy-server/tabpy_server/psws/callbacks.py +++ b/tabpy-server/tabpy_server/psws/callbacks.py @@ -4,6 +4,7 @@ from tornado import gen +from tabpy_server.app.SettingsParameters import SettingsParameters from tabpy_server.common.messages import ( LoadObject, DeleteObjects, ListObjects, ObjectList) from tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files @@ -49,7 +50,7 @@ def init_ps_server(settings, tabpy_state): try: object_version = obj_info['version'] get_query_object_path( - settings['state_file_path'], + settings[SettingsParameters.StateFilePath], object_name, object_version) except Exception as e: logger.error('Exception encounted when downloading object: %s' @@ -72,7 +73,7 @@ def init_model_evaluator(settings, tabpy_state, python_service): object_version = obj_info['version'] object_type = obj_info['type'] object_path = get_query_object_path( - settings['state_file_path'], + settings[SettingsParameters.StateFilePath], object_name, object_version) logger.info('Load endpoint: %s, version: %s, type: %s' % @@ -113,7 +114,7 @@ def _get_latest_service_state(settings, endpoint_info['version'] != existing_endpoint['version']: # Either a new endpoint or new endpoint version path_to_new_version = get_query_object_path( - settings['state_file_path'], + settings[SettingsParameters.StateFilePath], endpoint_name, endpoint_info['version']) endpoint_type = endpoint_info.get('type', 'model') diff[endpoint_name] = (endpoint_type, endpoint_info['version'], @@ -137,7 +138,8 @@ def _get_latest_service_state(settings, def on_state_change(settings, tabpy_state, python_service): try: logger.info("Loading state from state file") - config = util._get_state_from_file(settings['state_file_path']) + config = util._get_state_from_file( + settings[SettingsParameters.StateFilePath]) new_ps_state = TabPyState(config=config, settings=settings) (has_changes, changes) = _get_latest_service_state(settings, @@ -158,7 +160,8 @@ def on_state_change(settings, tabpy_state, python_service): python_service.manage_request(DeleteObjects([object_name])) - cleanup_endpoint_files(object_name, settings['upload_dir']) + cleanup_endpoint_files(object_name, + settings[SettingsParameters.UploadDir]) else: endpoint_info = new_endpoints[object_name] @@ -177,7 +180,7 @@ def on_state_change(settings, tabpy_state, python_service): # cleanup old version of endpoint files if object_version > 2: cleanup_endpoint_files( - object_name, settings['upload_dir'], [ + object_name, settings[SettingsParameters.UploadDir], [ object_version, object_version - 1]) except Exception as e: diff --git a/tabpy-tools/tabpy_tools/rest.py b/tabpy-tools/tabpy_tools/rest.py index 362071eb..dfb943b5 100755 --- a/tabpy-tools/tabpy_tools/rest.py +++ b/tabpy-tools/tabpy_tools/rest.py @@ -3,7 +3,7 @@ import requests from requests.auth import HTTPBasicAuth from re import compile -import json +import json as json from collections import MutableMapping as _MutableMapping @@ -141,8 +141,10 @@ def PUT(self, url, data, timeout=None): return response.json() def DELETE(self, url, data, timeout=None): - """Issues a DELETE request to the URL with the data specified. Returns an - object that is parsed from the response JSON.""" + ''' + Issues a DELETE request to the URL with the data specified. Returns an + object that is parsed from the response JSON. + ''' if data is not None: data = json.dumps(data) diff --git a/tabpy-tools/tools_tests/test_rest.py b/tabpy-tools/tools_tests/test_rest.py index c4d9cd70..f54d61ce 100644 --- a/tabpy-tools/tools_tests/test_rest.py +++ b/tabpy-tools/tools_tests/test_rest.py @@ -223,7 +223,8 @@ def setUp(self): nw.DELETE.return_value = 'DELETE' self.sc = ServiceClient('endpoint/', network_wrapper=nw) - self.scClientDoesNotEndWithSlash = ServiceClient('endpoint', network_wrapper=nw) + self.scClientDoesNotEndWithSlash =\ + ServiceClient('endpoint', network_wrapper=nw) def test_GET(self): self.assertEqual(self.sc.GET('test'), 'GET')