diff --git a/LICENSE b/LICENSE index 97ff689..25e1779 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2015-2022 Flávio Gonçalves Garcia + Copyright 2015-2024 Flavio Garcia Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/docs/conf.py b/docs/conf.py index 7144fdf..ad3d5e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,7 +57,7 @@ # The short X.Y version. version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.9.4' +release = '0.9.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/releases.rst b/docs/releases.rst index 6743cde..411a2f5 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -4,6 +4,8 @@ Release notes .. toctree:: :maxdepth: 2 + releases/v0.9.5 + releases/v0.9.4 releases/v0.9.3 releases/v0.9.0 releases/v0.2.17 diff --git a/docs/releases/v0.9.5.rst b/docs/releases/v0.9.5.rst new file mode 100644 index 0000000..ae3dcf5 --- /dev/null +++ b/docs/releases/v0.9.5.rst @@ -0,0 +1,32 @@ +What's new in Firenado 0.9.5 +============================ + +Apr 23, 2024 +------------ + +We are pleased to announce the release of Firenado 0.9.5. + +This release fix the sqlalchemy connection ping rolling back transactions while +using 2.x style ORM. + +It was also added a test case for Services to the project. + +Here are the highlights: + +Bug Fixes +~~~~~~~~~ + + * When creating a new sqlalchemy session, get default values from the data source configuration `#448 `_ + * Sqlalchemy data source ping connection will rollback every transaction `#449 `_ + * Fix requirements resolution while building distribution files `#451 `_ + +Features +~~~~~~~~ + + * Create a test case for services `#450 `_ + * Disable sqlalchemy connection_ping by default `#452 `_ + +Refactory +~~~~~~~~~ + + * Move isolation_level to the root of a sqlalchemy data source configuration `#447 `_ diff --git a/examples/testapp/app.py b/examples/testapp/app.py index 5efc48d..31b2138 100644 --- a/examples/testapp/app.py +++ b/examples/testapp/app.py @@ -68,10 +68,10 @@ def initialize(self): data_source_conf = { 'connector': "sqlalchemy", 'url': "mysql+pymysql://root@localhost:3306/test", + # 'isolation_level': 'REPEATABLE READ', 'pool': { 'size': 10, 'max_overflow': 10, - # 'isolation_level': 'REPEATABLE READ', # 'pool_recycle': 400 } } @@ -83,19 +83,22 @@ def initialize(self): @service.served_by(services.UserService) def install(self): + from sqlalchemy.engine import Engine """ Installing test database """ - from firenado.util.sqlalchemy_util import Base + from testapp.models import Base print('Installing Testapp App...') print('Creating App ...') - engine = self.application.get_data_source( + engine: Engine = self.application.get_data_source( 'test').engine engine.echo = True - # Dropping all - # TODO Not to drop all if something is installed right? - Base.metadata.drop_all(engine) - # Creating database - Base.metadata.create_all(engine) + with engine.connect() as conn: + # Dropping all + # TODO Not to drop all if something is installed right? + Base.metadata.drop_all(conn) + # Creating database + Base.metadata.create_all(conn) + conn.commit() self.user_service.create({ 'username': "Test", 'first_name': "Test", diff --git a/examples/testapp/handlers.py b/examples/testapp/handlers.py index 6fc36b7..bc12881 100644 --- a/examples/testapp/handlers.py +++ b/examples/testapp/handlers.py @@ -1,6 +1,6 @@ # -*- coding: UTF-8 -*- # -# Copyright 2015-2023 Flavio Garcia +# Copyright 2015-2024 Flavio Garcia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .services import LoginService, UserService + import firenado.conf from firenado import security, service, tornadoweb from firenado.util.sqlalchemy_util import base_to_dict @@ -115,6 +117,9 @@ def get(self): class LoginHandler(AuthHandler, tornadoweb.TornadoHandler): + login_service: LoginService + user_service: UserService + def get(self): default_login = firenado.conf.app['login']['urls']['default'] errors = {} @@ -126,8 +131,9 @@ def get(self): self.render("login.html", errors=errors, login_url=default_login) - @service.served_by("testapp.services.LoginService") - @service.served_by("testapp.services.UserService") + # you can user either the class rererence or string + @service.with_service(LoginService) + @service.with_service("testapp.services.UserService") def post(self): self.session.delete('login_errors') default_login = firenado.conf.app['login']['urls']['default'] diff --git a/examples/testapp/models.py b/examples/testapp/models.py index 5ae21d5..f1edccf 100644 --- a/examples/testapp/models.py +++ b/examples/testapp/models.py @@ -13,8 +13,7 @@ # limitations under the License. from datetime import datetime -from sqlalchemy import String -from sqlalchemy.types import DateTime +from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.sql import text @@ -25,11 +24,12 @@ class Base(DeclarativeBase): class UserBase(Base): __tablename__ = "users" + __table_args__ = { + 'mysql_engine': "InnoDB", + 'mysql_charset': "utf8", + } - mysql_engine = "MyISAM" - mysql_charset = "utf8" - - id: Mapped[int] = mapped_column(primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) username: Mapped[str] = mapped_column(String(150), nullable=False) first_name: Mapped[str] = mapped_column(String(150), nullable=False) last_name: Mapped[str] = mapped_column(String(150), nullable=False) diff --git a/examples/testapp/services.py b/examples/testapp/services.py index b6e794f..c7a1492 100644 --- a/examples/testapp/services.py +++ b/examples/testapp/services.py @@ -1,4 +1,4 @@ -# Copyright 2015-2023 Flávio Gonçalves Garcia +# Copyright 2015-2024 Flavio Garcia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,16 +37,17 @@ def create(self, user_data, **kwargs): user.last_name = user_data['last_name'] user.password = password_digest(user_data['password']) user.email = user_data['email'] - session = self.get_data_source('test').session - session.add(user) - session.commit() + with session.begin(): + session.add(user) return user @with_session(data_source="test") - def by_username(self, username, **kwargs): + def by_username(self, username, **kwargs) -> UserBase: session: Session = kwargs.get("session") + session.begin() stmt = select(UserBase).where(UserBase.username == username) - user = session.scalars(stmt).one() + user = session.scalars(stmt).one_or_none() + session.flush() return user @@ -68,9 +69,8 @@ def is_valid(self, username, password): """ user = self.user_service.by_username(username) + print(user) if user: if user.password == password_digest(password): return True return False - - diff --git a/firenado/__init__.py b/firenado/__init__.py index bb30429..fa2d367 100644 --- a/firenado/__init__.py +++ b/firenado/__init__.py @@ -18,7 +18,7 @@ """The Firenado Framework""" __author__ = "Flavio Garcia " -__version__ = (0, 9, 4) +__version__ = (0, 9, 5) __licence__ = "Apache License V2.0" diff --git a/firenado/config.py b/firenado/config.py index 64e5db0..08ffdd4 100644 --- a/firenado/config.py +++ b/firenado/config.py @@ -1,6 +1,4 @@ -# -*- coding: UTF-8 -*- -# -# Copyright 2015-2023 Flavio Garcia +# Copyright 2015-2024 Flavio Garcia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +13,7 @@ # limitations under the License. from cartola import sysexits -from cartola.config import load_yaml_file, log_level_from_string +from cartola.config import get_from_dict, load_yaml_file, log_level_from_string import logging import os @@ -70,46 +68,13 @@ def get_config_from_package(package): :param package: A package string. :return: A config dict with class and module. """ - package_x = package.split('.') + package_x = package.split(".") package_conf = {} package_conf['class'] = package_x[-1] - package_conf['module'] = '.'.join(package_x[:-1][:]) + package_conf['module'] = ".".join(package_x[:-1][:]) return package_conf -def get_class_from_name(name): - """ Return a class reference from the class name provided as a parameter. - Class name must be the full class reference, in another words, the module - with the class absolute reference. - - Example: - >>> get_class_from_name("my.module.Myclass") - - :param basestring name: Class absolute reference. - :return: The class resolved from the absolute reference name provided. - """ - return get_class_from_module( - ".".join(name.split(".")[:-1]), - name.split(".")[-1] - ) - - -def get_class_from_module(module, class_name): - """ Returns a class from a module and a class name parameters. - This function is used by get_class_from_config and get_class_from_name. - - Example: - >>> get_class_from_module("my.module", "MyClass") - - :param basestring module: The module name. - :param basestring class_name: The class name. - :return: The class resolved by the module and class name provided. - """ - import importlib - module = importlib.import_module(module) - return getattr(module, class_name) - - def get_class_from_config(config, index="class"): """ Return a class from a config dict bit containing the indexes module and class. @@ -128,7 +93,7 @@ def get_class_from_config(config, index="class"): :param basestring index: Index to be used to get the class name :return: The class resolved at the module referred into the config. """ - return get_class_from_module(config['module'], config[index]) + return get_from_dict(config, module_index="module", attr_index=index) def process_config(config, config_data): @@ -235,53 +200,50 @@ def process_app_config_section(config, app_config): configuration data from the config_data. :param app_config: App section from a config data dict. """ - if 'addresses' in app_config: + if "addresses" in app_config: config.app['addresses'] = app_config['addresses'] - if 'component' in app_config: + if "component" in app_config: config.app['component'] = app_config['component'] - if 'data' in app_config: - if 'sources' in app_config['data']: - config.app['data']['sources'] = app_config['data']['sources'] - if 'id' in app_config: + if "data" in app_config and "sources" in app_config['data']: + config.app['data']['sources'] = app_config['data']['sources'] + if "id" in app_config: config.app['id'] = app_config['id'] - if 'login' in app_config: - if 'urls' in app_config['login']: - if app_config['login']['urls']: - for url in app_config['login']['urls']: - config.app['login']['urls'][url['name']] = url['value'] - if 'pythonpath' in app_config: + if "login" in app_config and "urls" in app_config['login']: + if len(app_config['login']['urls']) > 0: + for url in app_config['login']['urls']: + config.app['login']['urls'][url['name']] = url['value'] + if "pythonpath" in app_config: config.app['pythonpath'] = app_config['pythonpath'] - if 'port' in app_config: + if "port" in app_config: config.app['port'] = app_config['port'] - if 'process' in app_config: - if 'num_processes' in app_config['process']: - config.app['process']['num_processes'] = app_config[ + if "process" in app_config and "num_processes" in app_config['process']: + config.app['process']['num_processes'] = app_config[ 'process']['num_processes'] - if 'url_root_path' in app_config: + if "url_root_path" in app_config: root_url = app_config['url_root_path'].strip() if root_url[0] == "/": root_url = root_url[1:] if root_url == "": root_url = None config.app['url_root_path'] = root_url - if 'settings' in app_config: + if "settings" in app_config: config.app['settings'] = app_config['settings'] - if 'socket' in app_config: + if "socket" in app_config: config.app['socket'] = app_config['socket'] - if 'static_path' in app_config: + if "static_path" in app_config: config.app['static_path'] = app_config['static_path'] - if 'static_url_prefix' in app_config: + if "static_url_prefix" in app_config: config.app['static_url_prefix'] = app_config['static_url_prefix'] - if 'type' in app_config: + if "type" in app_config: config.app['type'] = app_config['type'] - if 'types' in app_config: + if "types" in app_config: for app_type in app_config['types']: app_type['launcher'] = get_config_from_package( app_type['launcher']) config.app['types'][app_type['name']] = app_type - if 'xheaders' in app_config: + if "xheaders" in app_config: config.app['xheaders'] = app_config['xheaders'] - if 'wait_before_shutdown' in app_config: + if "wait_before_shutdown" in app_config: config.app['wait_before_shutdown'] = app_config['wait_before_shutdown'] @@ -293,7 +255,7 @@ def process_components_config_section(config, components_config): :param components_config: Data section from a config data dict. """ for component_config in components_config: - if 'id' not in component_config: + if "id" not in component_config: raise Exception('The component %s was defined without an id.' % component_config) component_id = component_config['id'] @@ -301,12 +263,12 @@ def process_components_config_section(config, components_config): config.components[component_id] = {} config.components[component_id]['enabled'] = False config.components[component_id]['config'] = {} - if 'class' in component_config: - class_config_x = component_config['class'].split('.') + if "class" in component_config: + class_config_x = component_config['class'].split(".") config.components[component_id]['class'] = class_config_x[-1] - config.components[component_id]['module'] = '.'.join( + config.components[component_id]['module'] = ".".join( class_config_x[:-1]) - if 'enabled' in component_config: + if "enabled" in component_config: config.components[component_id]['enabled'] = bool( component_config['enabled']) @@ -319,14 +281,13 @@ def process_data_config_section(config, data_config): configuration data from the config_data. :param data_config: Data configuration section from a config data dict. """ - if 'connectors' in data_config: + if "connectors" in data_config: for connector in data_config['connectors']: config.data['connectors'][ connector['name']] = get_config_from_package( connector['class']) - if 'sources' in data_config: - if data_config['sources']: - process_data_sources_config(config, data_config['sources']) + if "sources" in data_config and data_config['sources']: + process_data_sources_config(config, data_config['sources']) def process_data_sources_config_file(config, file): @@ -352,9 +313,9 @@ def process_log_config_section(config, log_config): configuration data from the config_data. :param log_config: Log section from a config data dict. """ - if 'format' in log_config: + if "format" in log_config: config.log['format'] = log_config['format'] - if 'level' in log_config: + if "level" in log_config: config.log['level'] = log_level_from_string(log_config['level']) @@ -365,7 +326,7 @@ def process_management_config_section(config, management_config): configuration data from the config_data. :param management_config: Management section from a config data dict. """ - if 'commands' in management_config: + if "commands" in management_config: for command in management_config['commands']: config.management['commands'].append(command) @@ -379,27 +340,25 @@ def process_session_config_section(config, session_config): dict. """ # Setting session type as file by default - config.session['type'] = 'file' - if 'enabled' in session_config: + config.session['type'] = "file" + if "enabled" in session_config: config.session['enabled'] = session_config['enabled'] - if 'type' in session_config: + if "type" in session_config: config.session['type'] = session_config['type'] - if config.session['type'] == 'file': - if 'path' in session_config: - config.session['file']['path'] = session_config['path'] - if config.session['type'] == 'redis': - if 'data' in session_config: - if 'source' in session_config['data']: - config.session['redis']['data']['source'] = session_config[ - 'data']['source'] - if 'handlers' in session_config: + if config.session['type'] == 'file' and "path" in session_config: + config.session['file']['path'] = session_config['path'] + if (config.session['type'] == 'redis' and "data" in session_config and + "source" in session_config['data']): + config.session['redis']['data']['source'] = session_config[ + 'data']['source'] + if "handlers" in session_config: for handler in session_config['handlers']: - handler_class_x = handler['class'].split('.') + handler_class_x = handler['class'].split(".") handler['class'] = handler_class_x[-1] - handler['module'] = '.'.join(handler_class_x[:-1][:]) + handler['module'] = ".".join(handler_class_x[:-1][:]) config.session['handlers'][handler['name']] = handler del config.session['handlers'][handler['name']]['name'] - if 'encoders' in session_config: + if "encoders" in session_config: for encoder in session_config['encoders']: encoder_class_x = encoder['class'].split('.') encoder['encoder'] = encoder_class_x[-1] @@ -407,26 +366,26 @@ def process_session_config_section(config, session_config): encoder['module'] = '.'.join(encoder_class_x[:-1][:]) config.session['encoders'][encoder['name']] = encoder del config.session['encoders'][encoder['name']]['name'] - if 'id_generators' in session_config: + if "id_generators" in session_config: for generator in session_config['id_generators']: - generator_ref_x = generator['function'].split('.') + generator_ref_x = generator['function'].split(".") generator['function'] = generator_ref_x[-1] - generator['module'] = '.'.join(generator_ref_x[:-1][:]) + generator['module'] = ".".join(generator_ref_x[:-1][:]) config.session['id_generators'][generator['name']] = generator del config.session['id_generators'][generator['name']]['name'] - if 'name' in session_config: + if "name" in session_config: config.session['name'] = session_config['name'] - if 'life_time' in session_config: + if "life_time" in session_config: config.session['life_time'] = session_config['life_time'] - if 'callback_hiccup' in session_config: + if "callback_hiccup" in session_config: config.session['callback_hiccup'] = session_config['callback_hiccup'] - if 'callback_time' in session_config: + if "callback_time" in session_config: config.session['callback_time'] = session_config['callback_time'] - if 'prefix' in session_config: + if "prefix" in session_config: config.session['prefix'] = session_config['prefix'] - if 'purge_limit' in session_config: + if "purge_limit" in session_config: config.session['purge_limit'] = session_config['purge_limit'] - if 'encoder' in session_config: + if "encoder" in session_config: if session_config['encoder'] in config.session['encoders']: config.session['encoder'] = session_config['encoder'] else: diff --git a/firenado/data.py b/firenado/data.py index 0251979..61557d7 100644 --- a/firenado/data.py +++ b/firenado/data.py @@ -1,4 +1,4 @@ -# Copyright 2015-2023 Flavio Garcia +# Copyright 2015-2024 Flavio Garcia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -134,34 +134,34 @@ def process_config(self, conf): class SqlalchemyConnector(Connector): + from sqlalchemy import Engine from sqlalchemy.orm import Session """ Connects a sqlalchemy engine to a data connected instance. Sqlalchemy support a big variety of relational database backends. The connection returned by this handler contains an engine and session created by sqlalchemy and the database backend name. """ + __name: str + __engine: Engine + __connection: dict - def __init__(self, data_connected): - super(SqlalchemyConnector, self).__init__(data_connected) - self.__name = None + def configure(self, name, conf): + from sqlalchemy import Connection, create_engine + from sqlalchemy import exc, event, select + self.__name = name self.__connection = { 'backend': None, + 'ping': False, 'session': { + 'autobegin': True, 'autoflush': True, 'expire_on_commit': True, 'info': None } } - self.__engine = None - - def configure(self, name, conf): - self.__name = name - from sqlalchemy import create_engine - from sqlalchemy import exc, event, select # We will set the isolation level to READ UNCOMMITTED by default # to avoid the "cache" effect sqlalchemy has without this option. # Solution from: http://bit.ly/2bDq0Nv - # TODO: Get the isolation level from data source conf engine_params = { 'isolation_level': "READ UNCOMMITTED" } @@ -176,15 +176,20 @@ def configure(self, name, conf): if conf['future']: engine_params['future'] = True + if "isolation_level" in conf: + if conf['isolation_level']: + engine_params['isolation_level'] = conf['isolation_level'] + + if "ping" in conf: + if conf['ping']: + engine_params['ping'] = conf['ping'] + if "pool" in conf: if "class" in conf['pool']: engine_params['pool_class'] = conf['pool']['class'] if isinstance(engine_params['pool_class'], str): engine_params['pool_class'] = config.get_from_string( engine_params['pool_class']) - if "isolation_level" in conf['pool']: - engine_params['isolation_level'] = conf['pool'][ - 'isolation_level'] if "max_overflow" in conf['pool']: engine_params['max_overflow'] = conf['pool']['max_overflow'] if "pool_recycle" in conf['pool']: @@ -193,6 +198,9 @@ def configure(self, name, conf): engine_params['pool_size'] = conf['pool']['size'] if "session" in conf: + if "autobegin" in conf['session']: + self.__connection['session']['autobegin'] = conf['session'][ + 'autobegin'] if "autoflush" in conf['session']: self.__connection['session']['autoflush'] = conf['session'][ 'autoflush'] @@ -209,7 +217,9 @@ def configure(self, name, conf): self.__engine = create_engine(conf['url'], **engine_params) @event.listens_for(self.__engine, "engine_connect") - def ping_connection(connection, branch): + def ping_connection(conn: Connection, branch): + if not self.__connection['ping']: + return # Adding ping connection event handler as described at the # pessimistic disconnect section of: http://bit.ly/2c8Sm2t logger.debug("Pinging sqlalchemy connection.") @@ -221,14 +231,16 @@ def ping_connection(connection, branch): return # turn off "close with result". This flag is only used with # "connectionless" execution, otherwise will be False in any case - save_should_close_with_result = connection.should_close_with_result - connection.should_close_with_result = False + save_should_close_with_result = conn.should_close_with_result + conn.should_close_with_result = False try: # run a SELECT 1. use a core select() so that # the SELECT of a scalar value without a table is # appropriately formatted for the backend logger.debug("Testing sqlalchemy connection.") - connection.scalar(select(1)) + conn.begin() + conn.scalar(select(1)) + conn.commit() except exc.DBAPIError as err: logger.warning(err) logger.warning("Firenado will try to reestablish the data " @@ -244,13 +256,15 @@ def ping_connection(connection, branch): # The disconnect detection here also causes the whole # connection pool to be invalidated so that all stale # connections are discarded. - connection.scalar(select([1])) + conn.begin() + conn.scalar(select(1)) + conn.commit() logger.warning("Data source connection reestablished.") else: raise finally: # restore "close with result" - connection.should_close_with_result = ( + conn.should_close_with_result = ( save_should_close_with_result) logger.info("Connecting to the database using the engine: %s.", self.__engine) @@ -273,19 +287,24 @@ def get_a_session(self, **kwargs) -> Session: Default parameters based on: https://bit.ly/3MjWDzF :param dict kwargs: - :key bool autoflush: Default to True - :key bool expire_on_commit: Default to False + :key bool autobegin: Default to False + :key bool autoflush: Default to False + :key bool expire_on_commit: Default to True :key dict info: Default to None :return Session: """ - autoflush = kwargs.get("autoflush", True) - expire_on_commit = kwargs.get("expire_on_commit", True) - info = kwargs.get("info") + session_config = self.__connection['session'] + autobegin = kwargs.get("autobegin", session_config['autobegin']) + autoflush = kwargs.get("autoflush", session_config['autoflush']) + expire_on_commit = kwargs.get("expire_on_commit", + session_config['expire_on_commit']) + info = kwargs.get("info", session_config['info']) from sqlalchemy.orm import sessionmaker - Session = sessionmaker(bind=self.__engine, autoflush=autoflush, - expire_on_commit=expire_on_commit, info=info) - return Session() + maker = sessionmaker(bind=self.__engine, autoflush=autoflush, + expire_on_commit=expire_on_commit, info=info, + autobegin=autobegin) + return maker() @property def backend(self): diff --git a/firenado/session.py b/firenado/session.py index f1d2191..1f376d7 100644 --- a/firenado/session.py +++ b/firenado/session.py @@ -190,15 +190,15 @@ def __init__(self, engine, data=None, sess_id=None): self.__changed = False def clear(self): - """ Clear all data stored into the session. This is not - renewing/creating a new session id. If you want that use + """ Clear all data stored into the session. This is not + renewing/creating a new session id. If you want that use session.destroy() """ self.__data.clear() self.__changed = True def destroy(self, request_handler): - """ Clearing session data and marking the session to be - renewed at the end of the request. """ + """ Clearing session data and marking the session to be renewed at the + end of the request. """ self.clear() self.__destroyed = True self.__engine.store_session(request_handler) diff --git a/firenado/sqlalchemy.py b/firenado/sqlalchemy.py index c98d9a0..3505ebd 100644 --- a/firenado/sqlalchemy.py +++ b/firenado/sqlalchemy.py @@ -133,6 +133,8 @@ def wrapper(self, *method_args, **method_kwargs): if not session: logger.warning("No session was resolved.") logger.debug("Closing session %s.", session) + if hasattr(session, "autoflush") and not session.autoflush: + session.flush() session.close() return result return wrapper diff --git a/firenado/testing.py b/firenado/testing.py index 56c5797..c7b366b 100644 --- a/firenado/testing.py +++ b/firenado/testing.py @@ -13,9 +13,11 @@ # limitations under the License. import asyncio +from firenado.service import FirenadoService from firenado.launcher import ProcessLauncher, TornadoLauncher from tornado.testing import (bind_unused_port, AsyncTestCase, AsyncHTTPTestCase) +from unittest import TestCase def get_event_loop(): @@ -28,6 +30,35 @@ def get_event_loop(): return loop if loop else asyncio.new_event_loop() +class ServiceTestCase(TestCase): + + def setUp(self): + """ Call the configure data connection method + """ + self.configure_data_connected() + + def configure_data_connected(self): + """Should be overridden with the configutation of the data connected + instance + """ + raise NotImplementedError() + + @property + def data_connected(self): + """Should be overridden by subclasses to return a data connected + instance + """ + raise NotImplementedError() + + +class TestableService(FirenadoService): + """ Serves a data connected method directly. + When decorating a data connected directly the service must return the + consumer. + """ + pass + + class TornadoAsyncTestCase(AsyncTestCase): @property diff --git a/requirements/basic.txt b/requirements/basic.txt index ea60f0e..31f14c9 100644 --- a/requirements/basic.txt +++ b/requirements/basic.txt @@ -1,3 +1,3 @@ cartola>=0.18 -taskio==0.0.6 +taskio==0.0.7 tornado==6.4 diff --git a/requirements/sqlalchemy.txt b/requirements/sqlalchemy.txt index d4ff7b1..4a25639 100644 --- a/requirements/sqlalchemy.txt +++ b/requirements/sqlalchemy.txt @@ -1 +1 @@ -sqlalchemy>=2.0.23 +sqlalchemy>=2.0.29 diff --git a/setup.py b/setup.py index 4267e03..ac06fd6 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright 2015-2023 Flavio Garcia +# Copyright 2015-2024 Flavio Garcia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,35 +16,30 @@ import firenado from setuptools import setup, find_packages -import sys +import os -try: - # for pip >= 10 - from pip._internal.req import parse_requirements -except ImportError: - # for pip <= 9.0.3 - print("error: Upgrade to a pip version newer than 10. Run \"pip install " - "--upgrade pip\".") - sys.exit(1) +with open("README.md", "r") as fh: + long_description = fh.read() # Solution from http://bit.ly/29Yl8VN def resolve_requires(requirements_file): - try: - requirements = parse_requirements("./%s" % requirements_file, - session=False) - return [str(ir.req) for ir in requirements] - except AttributeError: - # for pip >= 20.1.x - # Need to run again as the first run was ruined by the exception - requirements = parse_requirements("./%s" % requirements_file, - session=False) - # pr stands for parsed_requirement - return [str(pr.requirement) for pr in requirements] - + requires = [] + if os.path.isfile(f"./{requirements_file}"): + file_dir = os.path.dirname(f"./{requirements_file}") + with open(f"./{requirements_file}") as f: + for raw_line in f.readlines(): + line = raw_line.strip().replace("\n", "") + if len(line) > 0: + if line.startswith("-r "): + partial_file = os.path.join(file_dir, line.replace( + "-r ", "")) + partial_requires = resolve_requires(partial_file) + requires = requires + partial_requires + continue + requires.append(line) + return requires -with open("README.md", "r") as fh: - long_description = fh.read() setup( name="Firenado", diff --git a/tests/config_test.py b/tests/config_test.py index 096ae0f..6f7b579 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python -# -# Copyright 2015-2023 Flavio Garcia +# Copyright 2015-2024 Flavio Garcia # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from cartola.config import get_from_module, get_from_string import unittest -from firenado.config import (get_class_from_config, get_class_from_module, - get_class_from_name) +from firenado.config import get_class_from_config from firenado.session import SessionEngine @@ -40,10 +38,10 @@ def test_get_class_from_config(self): def test_get_class_from_name(self): """ Getting a class from the full class name.""" - result = get_class_from_name("firenado.session.SessionEngine") + result = get_from_string("firenado.session.SessionEngine") self.assertTrue(result == SessionEngine) def test_get_class_from_module(self): """ Getting a class from a given module and class name parameters. """ - result = get_class_from_module("firenado.session", "SessionEngine") + result = get_from_module("firenado.session", "SessionEngine") self.assertTrue(result == SessionEngine) diff --git a/tests/service_test.py b/tests/service_test.py index 9664085..fb39ce8 100644 --- a/tests/service_test.py +++ b/tests/service_test.py @@ -14,15 +14,7 @@ from firenado.data import DataConnectedMixin from firenado.service import with_service, FirenadoService -import unittest - - -class TestableServiceDataConnected(FirenadoService): - """ Serves a data connected method directly. - When decorating a data connected directly the service must return the - consumer. - """ - pass +from firenado.testing import ServiceTestCase, TestableService class TestableSession(object): @@ -69,35 +61,24 @@ def resolve_session(self): class TestableDataConnected(DataConnectedMixin): - """ Data connected mock object. This object holds the data sources to be + """ Data connected dummy object. This object holds the data sources to be used in the test cases. """ - testable_service_data_connected: TestableServiceDataConnected + testable_service: TestableService def __init__(self): self.data_sources['datasource1'] = TestableDataSource("DataSource1") self.data_sources['datasource2'] = TestableDataSource("DataSource2") - @with_service(TestableServiceDataConnected) + @with_service(TestableService) def get_service_data_sources_directly(self): - return self.testable_service_data_connected.get_data_sources() - - -class TestableService(FirenadoService): - """ Service that decorates the instance to be served directly and - indirectly thought MockTestServiceRecursion. - When decorating directly data connected and data sources will be returned - in one interaction. - When decorating indirectly MockTestServiceRecursion will be used to return - the data connected instance and data sources. - """ - pass + return self.testable_service.get_data_sources() class RecursiveService(FirenadoService): """ This service will be used to return the data connected reference - and data source, during the recursive test, delegating to MockTestService. + and data source, during the recursive test, delegating to TestableService. """ testable_service: TestableService @@ -111,7 +92,7 @@ def get_data_connected_recursively(self): return self.testable_service.data_connected -class ServedByInstance(object): +class ServedInstance(object): """ Class with methods to be decorated with the served_by decorator. """ @@ -122,13 +103,13 @@ def __init__(self, data_connected): self.data_connected = data_connected @with_service(TestableService) - def do_served_by_class(self): + def do_with_service_by_class(self): """ Method to be decorated with served_by with a class reference """ pass @with_service("tests.service_test.TestableService") - def do_served_by_string(self): + def do_with_service_by_string(self): """ Method to be decorated with served_by with a string as class reference """ @@ -162,37 +143,36 @@ def get_data_connected(self): return self.data_connected -class ServiceTestCase(unittest.TestCase): +class WithServiceTestCase(ServiceTestCase): - def setUp(self): + def configure_data_connected(self): """ Setting up an object that has firenado.core.service.served_by decorators on some methods. """ self.data_connected_instance = TestableDataConnected() - self.served_by_instance = ServedByInstance( - self.data_connected_instance) + self.served_instance = ServedInstance(self.data_connected_instance) - def test_served_by_class_reference(self): - self.assertFalse(hasattr(self.served_by_instance, 'testable_service')) - self.served_by_instance.do_served_by_class() - self.assertTrue(hasattr(self.served_by_instance, 'testable_service')) + def test_with_service_class_reference(self): + self.assertFalse(hasattr(self.served_instance, "testable_service")) + self.served_instance.do_with_service_by_class() + self.assertTrue(hasattr(self.served_instance, "testable_service")) self.assertTrue(isinstance( - self.served_by_instance.testable_service, TestableService)) + self.served_instance.testable_service, TestableService)) - def test_served_by_class_name_string(self): - self.assertFalse(hasattr(self.served_by_instance, 'testable_service')) - self.served_by_instance.do_served_by_string() - self.assertTrue(hasattr(self.served_by_instance, 'testable_service')) + def test_with_service_class_name_string(self): + self.assertFalse(hasattr(self.served_instance, 'testable_service')) + self.served_instance.do_with_service_by_string() + self.assertTrue(hasattr(self.served_instance, 'testable_service')) self.assertEqual( - self.served_by_instance.testable_service.__class__.__name__, + self.served_instance.testable_service.__class__.__name__, TestableService.__name__) def test_data_connected_from_service(self): - data_connected = self.served_by_instance.get_data_connected() + data_connected = self.served_instance.get_data_connected() self.assertEqual(data_connected, self.data_connected_instance) def test_data_connected_from_service_recursively(self): - data_connected = (self.served_by_instance. + data_connected = (self.served_instance. get_service_data_connected_recursively()) self.assertEqual(data_connected, self.data_connected_instance) @@ -202,7 +182,7 @@ def test_data_connected_from_service_none(self): self.assertIsNone(data_connected) def test_get_data_source_from_service(self): - data_sources = self.served_by_instance.get_service_data_sources() + data_sources = self.served_instance.get_service_data_sources() self.assertTrue(len(data_sources) == 2) self.assertEqual(data_sources['datasource1'].name, "DataSource1") self.assertEqual(data_sources['datasource2'].name, "DataSource2") @@ -215,7 +195,7 @@ def test_get_data_source_from_data_connected(self): self.assertEqual(data_sources['datasource2'].name, "DataSource2") def test_get_data_source_from_service_recursively(self): - data_sources = (self.served_by_instance. + data_sources = (self.served_instance. get_service_data_sources_recursively()) self.assertTrue(len(data_sources) == 2) self.assertEqual(data_sources['datasource1'].name, "DataSource1") diff --git a/tests/sqlalchemy_test.py b/tests/sqlalchemy_test.py index 84087bc..26e68bb 100644 --- a/tests/sqlalchemy_test.py +++ b/tests/sqlalchemy_test.py @@ -13,9 +13,10 @@ # limitations under the License. from datetime import datetime -from tests.service_test import TestableDataConnected, ServedByInstance +from tests.service_test import TestableDataConnected, ServedInstance from firenado.sqlalchemy import base_to_dict, with_session from firenado.service import FirenadoService, with_service +from firenado.testing import ServiceTestCase from sqlalchemy import String from sqlalchemy.types import DateTime from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -43,7 +44,7 @@ class TestBase(Base): server_default=text("now()")) -class MockSessionedService(FirenadoService): +class DummySessionedService(FirenadoService): @with_session def resolve_from_default_data_source(self, **kwargs): @@ -96,23 +97,22 @@ def test_base_to_dict_parametrized(self): self.assertTrue("modified" not in dict_from_base) -class SessionedTestCase(unittest.TestCase): +class SessionedTestCase(ServiceTestCase): - mock_sessioned_service: MockSessionedService + dummy_sessioned_service: DummySessionedService - def setUp(self): + def configure_data_connected(self): """ Setting up an object that has firenado.core.service.served_by decorators on some methods. """ self.data_connected_instance = TestableDataConnected() - self.served_by_instance = ServedByInstance( - self.data_connected_instance) + self.served_by_instance = ServedInstance(self.data_connected_instance) @property def data_connected(self): return self.served_by_instance.data_connected - @with_service(MockSessionedService) + @with_service(DummySessionedService) def test_sessioned_default_data_source(self): """ Method resolve_from_default_data_source is anoteded with sessioned and no parameter. The data source to be used is the one defined either @@ -122,7 +122,7 @@ def test_sessioned_default_data_source(self): As no session was provided the session will be closed """ resolved_kwargs = ( - self.mock_sessioned_service.resolve_from_default_data_source() + self.dummy_sessioned_service.resolve_from_default_data_source() ) self.assertEqual("datasource2", resolved_kwargs['data_source']) data_source = self.data_connected.get_data_source( @@ -132,7 +132,7 @@ def test_sessioned_default_data_source(self): resolved_kwargs['session'].name) self.assertFalse(resolved_kwargs['session'].is_oppened) - @with_service(MockSessionedService) + @with_service(DummySessionedService) def test_sessioned_default_data_source_my_session(self): """ Method resolve_from_default_data_source is anoteded with sessioned and no parameter. Instead of getting the session from the default data @@ -140,7 +140,7 @@ def test_sessioned_default_data_source_my_session(self): """ data_source = self.data_connected.get_data_source("datasource1") resolved_kwargs = ( - self.mock_sessioned_service.resolve_from_default_data_source( + self.dummy_sessioned_service.resolve_from_default_data_source( session=data_source.session ) ) @@ -151,14 +151,14 @@ def test_sessioned_default_data_source_my_session(self): resolved_kwargs['session'].name) self.assertTrue(resolved_kwargs['session'].is_oppened) - @with_service(MockSessionedService) + @with_service(DummySessionedService) def test_sessioned_from_data_source(self): """ Method resolve_from_data_source is anoteded with sessioned and datasource1 as parameter. It will be injected to the method kwargs session from datasource1. """ resolved_kwargs = ( - self.mock_sessioned_service.resolve_from_data_source() + self.dummy_sessioned_service.resolve_from_data_source() ) self.assertEqual("datasource1", resolved_kwargs['data_source']) data_source = self.data_connected.get_data_source( @@ -168,7 +168,7 @@ def test_sessioned_from_data_source(self): resolved_kwargs['session'].name) self.assertFalse(resolved_kwargs['session'].is_oppened) - @with_service(MockSessionedService) + @with_service(DummySessionedService) def test_sessioned_from_data_source_provide_session(self): """ Method resolve_from_default_data_source is anoteded with sessioned and no parameter. This time we provide the session and provided to @@ -176,7 +176,7 @@ def test_sessioned_from_data_source_provide_session(self): """ data_source = self.data_connected.get_data_source("datasource2") resolved_kwargs = ( - self.mock_sessioned_service.resolve_from_data_source( + self.dummy_sessioned_service.resolve_from_data_source( session=data_source.session, close=True ) ) @@ -187,7 +187,7 @@ def test_sessioned_from_data_source_provide_session(self): resolved_kwargs['session'].name) self.assertFalse(resolved_kwargs['session'].is_oppened) - @with_service(MockSessionedService) + @with_service(DummySessionedService) def test_sessioned_with_my_data_source_closing_connection(self): """ Method resolve_from_data_source is anoteded with sessioned and datasource1 as parameter. We're overwriting the data_source parameter @@ -195,7 +195,7 @@ def test_sessioned_with_my_data_source_closing_connection(self): to the method kwargs a session from data_source1. """ resolved_kwargs = ( - self.mock_sessioned_service.resolve_from_data_source( + self.dummy_sessioned_service.resolve_from_data_source( data_source="datasource2") ) self.assertEqual("datasource2", resolved_kwargs['data_source'])