From 62b6ff4789e3bdb3b92e347e457309712b3245ff Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 6 Jun 2020 11:15:41 +0200 Subject: [PATCH 01/55] Setup DataBaseResolver --- .../core/configuration/__init__.py | 24 ++- .../core/configuration/config_constants.py | 4 + investing_bot_framework/core/exceptions.py | 8 + investing_bot_framework/core/extensions.py | 3 + .../core/models/__init__.py | 1 + .../core/resolvers/__init__.py | 2 + .../core/resolvers/database_resolver.py | 184 ++++++++++++++++++ .../tests/core/resolvers/__init__.py | 0 .../core/resolvers/test_database_resolver.py | 31 +++ .../tests/resources/__init__.py | 13 ++ .../tests/resources/standard_settings.py | 10 + requirements.txt | 1 + 12 files changed, 268 insertions(+), 13 deletions(-) create mode 100644 investing_bot_framework/core/extensions.py create mode 100644 investing_bot_framework/core/models/__init__.py create mode 100644 investing_bot_framework/core/resolvers/database_resolver.py create mode 100644 investing_bot_framework/tests/core/resolvers/__init__.py create mode 100644 investing_bot_framework/tests/core/resolvers/test_database_resolver.py create mode 100644 investing_bot_framework/tests/resources/__init__.py create mode 100644 investing_bot_framework/tests/resources/standard_settings.py diff --git a/investing_bot_framework/core/configuration/__init__.py b/investing_bot_framework/core/configuration/__init__.py index 0a6481bd..7efb3279 100644 --- a/investing_bot_framework/core/configuration/__init__.py +++ b/investing_bot_framework/core/configuration/__init__.py @@ -5,8 +5,8 @@ from investing_bot_framework.core.exceptions import ImproperlyConfigured, OperationalException from investing_bot_framework.core.configuration.template import Template -from investing_bot_framework.core.configuration.config_constants import SETTINGS_MODULE_PATH_ENV_NAME, SETTINGS_STRATEGY_REGISTERED_APPS, \ - SETTINGS_DATA_PROVIDER_REGISTERED_APPS +from investing_bot_framework.core.configuration.config_constants import SETTINGS_MODULE_PATH_ENV_NAME, \ + SETTINGS_STRATEGY_REGISTERED_APPS, SETTINGS_DATA_PROVIDER_REGISTERED_APPS, BASE_DIR class TimeUnit(Enum): @@ -59,14 +59,17 @@ class BaseSettings: Base wrapper for settings module. It will load all the default settings for a given settings module """ - def __init__(self, settings_module: str = None) -> None: + def __init__(self) -> None: self._configured = False - self._settings_module = settings_module + self._settings_module = None - if self._settings_module is not None: - self.configure() + def configure(self, settings_module: str = None) -> None: + self._settings_module = settings_module - def configure(self) -> None: + if settings_module is None: + self.settings_module = os.environ.get(SETTINGS_MODULE_PATH_ENV_NAME) + else: + self.settings_module = settings_module if self.settings_module is None: raise ImproperlyConfigured("There is no settings module defined") @@ -100,7 +103,6 @@ def settings_module(self) -> str: @settings_module.setter def settings_module(self, settings_module: str) -> None: self._settings_module = settings_module - self.configure() @property def configured(self) -> bool: @@ -131,8 +133,4 @@ def get(self, key: str, default: Any = None) -> Any: return default -def resolve_settings(): - return BaseSettings(os.environ.get(SETTINGS_MODULE_PATH_ENV_NAME)) - - -settings = resolve_settings() +settings = BaseSettings() diff --git a/investing_bot_framework/core/configuration/config_constants.py b/investing_bot_framework/core/configuration/config_constants.py index 81345b23..edd0edcb 100644 --- a/investing_bot_framework/core/configuration/config_constants.py +++ b/investing_bot_framework/core/configuration/config_constants.py @@ -11,3 +11,7 @@ # Operational constants DEFAULT_MAX_WORKERS = 2 + +# Database related constants +BASE_DIR = 'BASE_DIR' +DATABASE_NAME = 'DATABASE_NAME' diff --git a/investing_bot_framework/core/exceptions.py b/investing_bot_framework/core/exceptions.py index 15ed74a8..c20ee3cf 100644 --- a/investing_bot_framework/core/exceptions.py +++ b/investing_bot_framework/core/exceptions.py @@ -12,3 +12,11 @@ class OperationalException(Exception): """ def __init__(self, message) -> None: super(OperationalException, self).__init__(message) + + +class DatabaseOperationalException(Exception): + """ + Class DatabaseOperationalException: Exception class indicating a problem occurred during usage of the database + """ + def __init__(self, message) -> None: + super(DatabaseOperationalException, self).__init__(message) diff --git a/investing_bot_framework/core/extensions.py b/investing_bot_framework/core/extensions.py new file mode 100644 index 00000000..a96f80c7 --- /dev/null +++ b/investing_bot_framework/core/extensions.py @@ -0,0 +1,3 @@ +from investing_bot_framework.core.resolvers import DatabaseResolver + +db = DatabaseResolver() diff --git a/investing_bot_framework/core/models/__init__.py b/investing_bot_framework/core/models/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/investing_bot_framework/core/models/__init__.py @@ -0,0 +1 @@ + diff --git a/investing_bot_framework/core/resolvers/__init__.py b/investing_bot_framework/core/resolvers/__init__.py index d265cc44..d087e7f9 100644 --- a/investing_bot_framework/core/resolvers/__init__.py +++ b/investing_bot_framework/core/resolvers/__init__.py @@ -1 +1,3 @@ from investing_bot_framework.core.resolvers.class_collector import ClassCollector +from investing_bot_framework.core.resolvers.database_resolver import DatabaseResolver + diff --git a/investing_bot_framework/core/resolvers/database_resolver.py b/investing_bot_framework/core/resolvers/database_resolver.py new file mode 100644 index 00000000..a2ae1842 --- /dev/null +++ b/investing_bot_framework/core/resolvers/database_resolver.py @@ -0,0 +1,184 @@ +import os + +from sqlalchemy import create_engine +from sqlalchemy.orm import Query, class_mapper, sessionmaker, scoped_session +from sqlalchemy.orm.exc import UnmappedClassError +from sqlalchemy.ext.declarative import declarative_base + +from investing_bot_framework.core.configuration import settings +from investing_bot_framework.core.configuration.config_constants import BASE_DIR, DATABASE_NAME +from investing_bot_framework.core.exceptions import DatabaseOperationalException + +from typing import Any +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.orm.exc import DetachedInstanceError +from sqlalchemy.exc import DatabaseError + + +class Model(object): + table_name = None + session = None + query_class = None + query = None + + @declared_attr + def __tablename__(cls): + + if cls.table_name is None: + return cls.__name__.lower() + return cls.table_name + + def save(self): + self.session.add(self) + self._flush() + return self + + def update(self, **kwargs): + + for attr, value in kwargs.items(): + setattr(self, attr, value) + return self.save() + + def delete(self): + self.session.delete(self) + self._flush() + + def _flush(self): + try: + self.session.flush() + except DatabaseError: + self.session.rollback() + raise + + def _repr(self, **fields: Any) -> str: + """ + Helper for __repr__ + """ + + field_strings = [] + at_least_one_attached_attribute = False + + for key, field in fields.items(): + try: + field_strings.append(f'{key}={field!r}') + except DetachedInstanceError: + field_strings.append(f'{key}=DetachedInstanceError') + else: + at_least_one_attached_attribute = True + + if at_least_one_attached_attribute: + return f"<{self.__class__.__name__}({','.join(field_strings)})>" + + return f"<{self.__class__.__name__} {id(self)}>" + + +class BaseQuery(Query): + """ + SQLAlchemy :class:`~sqlalchemy.orm.query.Query` subclass with convenience methods for querying in a web + application. This is the default :attr:`~Model.query` object used for models, and + exposed as :attr:`~SQLAlchemy.Query`. Override the query class for an individual model by subclassing + this and setting :attr:`~Model.query_class`. + """ + + def get_or_404(self, ident, description=None): + """Like :meth:`get` but aborts with 404 if not found instead of returning ``None``.""" + + rv = self.get(ident) + if rv is None: + raise Exception() + return rv + + def first_or_404(self, description=None): + """Like :meth:`first` but aborts with 404 if not found instead of returning ``None``.""" + + rv = self.first() + if rv is None: + raise Exception() + return rv + + +class _SessionProperty: + """ + Wrapper for session property of a Model + + To make sure that each thread gets an scoped session, a new scoped session is created if a new thread + accesses the session property of a Model. + """ + def __init__(self, db): + self.db = db + + def __get__(self, instance, owner): + return self.db.session + + +class _QueryProperty: + """ + Wrapper for query property of a Model + + This wrapper makes sure that each model gets a Query object with a correct session corresponding to its thread. + """ + def __init__(self, db): + self.db = db + + def __get__(self, instance, owner): + + try: + mapper = class_mapper(owner) + if mapper: + return owner.query_class(mapper, session=self.db.session) + + except UnmappedClassError: + return None + + +class DatabaseResolver: + + def __init__(self, query_class=BaseQuery, model_class=Model): + self._configured = False + self.Query = query_class + self.Model = self.make_declarative_base(model_class) + self.engine = None + self.session_factory = None + self.Session = None + self.database_path = None + + def configure(self): + self.initialize() + self.session_factory = sessionmaker(bind=self.engine) + self.Session = scoped_session(self.session_factory) + + if self.Model is None: + raise DatabaseOperationalException("Model is not defined") + + self.Model.session = _SessionProperty(self) + + if not getattr(self.Model, 'query_class', None): + self.Model.query_class = self.Query + + self.Model.query = _QueryProperty(self) + + def initialize(self): + base_dir = settings.get(BASE_DIR) + database_name = settings.get(DATABASE_NAME) + + if database_name is not None: + self.database_path = os.path.join(base_dir, database_name, '.sqlite3') + else: + self.database_path = os.path.join(base_dir, 'db.sqlite3') + + # Only create the database if not exist + if not os.path.isfile(self.database_path): + os.mknod(self.database_path) + + self.engine = create_engine('sqlite:////{}'.format(self.database_path)) + + @staticmethod + def make_declarative_base(model_class): + """ + Creates the declarative base that all models will inherit from. + + :param model_class: base model class to pass to :func:`~sqlalchemy.ext.declarative.declarative_base`. + """ + + model = declarative_base(cls=model_class) + return model diff --git a/investing_bot_framework/tests/core/resolvers/__init__.py b/investing_bot_framework/tests/core/resolvers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py new file mode 100644 index 00000000..40304c4a --- /dev/null +++ b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py @@ -0,0 +1,31 @@ +import os +from unittest import TestCase + +from investing_bot_framework.core.resolvers import DatabaseResolver +from investing_bot_framework.tests.resources import BaseTestMixin +from investing_bot_framework.core.configuration import settings +from investing_bot_framework.core.extensions import db + + +class TestDatabaseResolverConfiguration(TestCase, BaseTestMixin): + + def setUp(self) -> None: + self.initialize_environment() + self.db_resolver = DatabaseResolver() + + def test_configuration(self): + settings.configure() + db.configure() + + # Check if all properties are configured + self.assertIsNotNone(db.Session) + self.assertIsNotNone(db.engine) + self.assertIsNotNone(db.session_factory) + self.assertIsNotNone(db.database_path) + + def tearDown(self) -> None: + + if os.path.isfile(db.database_path): + os.remove(db.database_path) + + diff --git a/investing_bot_framework/tests/resources/__init__.py b/investing_bot_framework/tests/resources/__init__.py new file mode 100644 index 00000000..0013e49a --- /dev/null +++ b/investing_bot_framework/tests/resources/__init__.py @@ -0,0 +1,13 @@ +import os + +import investing_bot_framework.tests.resources.standard_settings + + +class BaseTestMixin: + + @staticmethod + def initialize_environment(): + os.environ.setdefault( + 'INVESTING_BOT_FRAMEWORK_SETTINGS_MODULE', 'investing_bot_framework.tests.resources.standard_settings' + ) + diff --git a/investing_bot_framework/tests/resources/standard_settings.py b/investing_bot_framework/tests/resources/standard_settings.py new file mode 100644 index 00000000..97e7135a --- /dev/null +++ b/investing_bot_framework/tests/resources/standard_settings.py @@ -0,0 +1,10 @@ +from pathlib import Path + +BOT_PROJECT_NAME = 'TEST_ALGORITHM' + +BASE_DIR = str(Path(__file__).parent) + +INSTALLED_DATA_PROVIDER_APPS = [] + +INSTALLED_STRATEGY_APPS = [] + diff --git a/requirements.txt b/requirements.txt index 92cd37ba..c384b3bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ requests==2.22.0 pandas==0.25.3 wrapt==1.11.2 colorama==0.4.3 +SQLAlchemy==1.3.13 From 022feabc0f35e3233811369e8fd3b6c9ad2e0244 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 6 Jun 2020 15:17:53 +0200 Subject: [PATCH 02/55] Add check if database is file --- .../tests/core/resolvers/test_database_resolver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py index 40304c4a..4ad89c72 100644 --- a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py +++ b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py @@ -22,6 +22,7 @@ def test_configuration(self): self.assertIsNotNone(db.engine) self.assertIsNotNone(db.session_factory) self.assertIsNotNone(db.database_path) + self.assertTrue(os.path.isfile(db.database_path)) def tearDown(self) -> None: From 77f5f47e9ab300eac5d36eabe0a709d832e5207b Mon Sep 17 00:00:00 2001 From: sadays Date: Wed, 10 Jun 2020 12:44:18 +0200 Subject: [PATCH 03/55] Add tests data base resolver --- .../core/models/__init__.py | 1 - .../core/resolvers/database_resolver.py | 142 +++++++++--------- .../core/resolvers/test_database_resolver.py | 38 ++++- .../tests/resources/utils.py | 10 ++ 4 files changed, 114 insertions(+), 77 deletions(-) delete mode 100644 investing_bot_framework/core/models/__init__.py create mode 100644 investing_bot_framework/tests/resources/utils.py diff --git a/investing_bot_framework/core/models/__init__.py b/investing_bot_framework/core/models/__init__.py deleted file mode 100644 index 8b137891..00000000 --- a/investing_bot_framework/core/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/investing_bot_framework/core/resolvers/database_resolver.py b/investing_bot_framework/core/resolvers/database_resolver.py index a2ae1842..12c57d16 100644 --- a/investing_bot_framework/core/resolvers/database_resolver.py +++ b/investing_bot_framework/core/resolvers/database_resolver.py @@ -1,25 +1,62 @@ import os +from typing import Any from sqlalchemy import create_engine -from sqlalchemy.orm import Query, class_mapper, sessionmaker, scoped_session +from sqlalchemy.orm import Query, class_mapper, sessionmaker, scoped_session, Session from sqlalchemy.orm.exc import UnmappedClassError -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.declarative import declarative_base, declared_attr +from sqlalchemy.orm.exc import DetachedInstanceError +from sqlalchemy.exc import DatabaseError + from investing_bot_framework.core.configuration import settings from investing_bot_framework.core.configuration.config_constants import BASE_DIR, DATABASE_NAME from investing_bot_framework.core.exceptions import DatabaseOperationalException -from typing import Any -from sqlalchemy.ext.declarative import declared_attr -from sqlalchemy.orm.exc import DetachedInstanceError -from sqlalchemy.exc import DatabaseError + +class _SessionProperty: + """ + Wrapper for session property of a Model + + To make sure that each thread gets an scoped session, a new scoped session is created if a new thread + accesses the session property of a Model. + """ + def __init__(self, db): + self.db = db + + def __get__(self, instance, owner): + return self.db.session + + +class _QueryProperty: + """ + Wrapper for query property of a Model + + This wrapper makes sure that each model gets a Query object with a correct session corresponding to its thread. + """ + def __init__(self, db): + self.db = db + + def __get__(self, instance, owner): + + try: + mapper = class_mapper(owner) + if mapper: + return owner.query_class(mapper, session=self.db.session) + + except UnmappedClassError: + return None class Model(object): table_name = None session = None query_class = None - query = None + _query = None + + @property + def query(self) -> Query: + return self._query @declared_attr def __tablename__(cls): @@ -72,71 +109,12 @@ def _repr(self, **fields: Any) -> str: return f"<{self.__class__.__name__} {id(self)}>" -class BaseQuery(Query): - """ - SQLAlchemy :class:`~sqlalchemy.orm.query.Query` subclass with convenience methods for querying in a web - application. This is the default :attr:`~Model.query` object used for models, and - exposed as :attr:`~SQLAlchemy.Query`. Override the query class for an individual model by subclassing - this and setting :attr:`~Model.query_class`. - """ - - def get_or_404(self, ident, description=None): - """Like :meth:`get` but aborts with 404 if not found instead of returning ``None``.""" - - rv = self.get(ident) - if rv is None: - raise Exception() - return rv - - def first_or_404(self, description=None): - """Like :meth:`first` but aborts with 404 if not found instead of returning ``None``.""" - - rv = self.first() - if rv is None: - raise Exception() - return rv - - -class _SessionProperty: - """ - Wrapper for session property of a Model - - To make sure that each thread gets an scoped session, a new scoped session is created if a new thread - accesses the session property of a Model. - """ - def __init__(self, db): - self.db = db - - def __get__(self, instance, owner): - return self.db.session - - -class _QueryProperty: - """ - Wrapper for query property of a Model - - This wrapper makes sure that each model gets a Query object with a correct session corresponding to its thread. - """ - def __init__(self, db): - self.db = db - - def __get__(self, instance, owner): - - try: - mapper = class_mapper(owner) - if mapper: - return owner.query_class(mapper, session=self.db.session) - - except UnmappedClassError: - return None - - class DatabaseResolver: - def __init__(self, query_class=BaseQuery, model_class=Model): + def __init__(self, query_class=Query, model_class=Model): self._configured = False self.Query = query_class - self.Model = self.make_declarative_base(model_class) + self._model = self.make_declarative_base(model_class) self.engine = None self.session_factory = None self.Session = None @@ -147,15 +125,15 @@ def configure(self): self.session_factory = sessionmaker(bind=self.engine) self.Session = scoped_session(self.session_factory) - if self.Model is None: + if self._model is None: raise DatabaseOperationalException("Model is not defined") - self.Model.session = _SessionProperty(self) + self._model.session = _SessionProperty(self) - if not getattr(self.Model, 'query_class', None): - self.Model.query_class = self.Query + if not getattr(self._model, 'query_class', None): + self._model.query_class = self.Query - self.Model.query = _QueryProperty(self) + self._model.query = _QueryProperty(self) def initialize(self): base_dir = settings.get(BASE_DIR) @@ -182,3 +160,21 @@ def make_declarative_base(model_class): model = declarative_base(cls=model_class) return model + + @property + def session(self) -> Session: + """ + Returns scoped session of an Session object + """ + return self.Session() + + @property + def model(self) -> Model: + return self._model + + def initialize_tables(self): + self._model.metadata.create_all(self.engine) + + + + diff --git a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py index 4ad89c72..974d706f 100644 --- a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py +++ b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py @@ -1,17 +1,21 @@ import os from unittest import TestCase +from sqlalchemy import Column, String, Integer -from investing_bot_framework.core.resolvers import DatabaseResolver -from investing_bot_framework.tests.resources import BaseTestMixin +from investing_bot_framework.tests.resources import BaseTestMixin, utils from investing_bot_framework.core.configuration import settings from investing_bot_framework.core.extensions import db +class TestModel(db.model): + id = Column(Integer, primary_key=True) + name = Column(String()) + + class TestDatabaseResolverConfiguration(TestCase, BaseTestMixin): def setUp(self) -> None: self.initialize_environment() - self.db_resolver = DatabaseResolver() def test_configuration(self): settings.configure() @@ -30,3 +34,31 @@ def tearDown(self) -> None: os.remove(db.database_path) +class TestDatabaseResolverModel(TestCase, BaseTestMixin): + + def setUp(self) -> None: + self.initialize_environment() + settings.configure() + db.configure() + db.initialize_tables() + + def test(self) -> None: + + model = TestModel(name=utils.random_string(10)) + model.save() + db.session.commit() + self.assertEqual(1, len(TestModel.query.all())) + + model = TestModel(name=utils.random_string(10)) + model.save() + db.session.commit() + self.assertEqual(2, len(TestModel.query.all())) + + def tearDown(self) -> None: + + if os.path.isfile(db.database_path): + os.remove(db.database_path) + + + + diff --git a/investing_bot_framework/tests/resources/utils.py b/investing_bot_framework/tests/resources/utils.py new file mode 100644 index 00000000..7fb043b7 --- /dev/null +++ b/investing_bot_framework/tests/resources/utils.py @@ -0,0 +1,10 @@ +import random +import string + +def random_string(n, spaces: bool = False): + + if spaces: + return ''.join(random.choice(string.ascii_lowercase + ' ') for _ in range(n)) + + return ''.join(random.choice(string.ascii_lowercase) for _ in range(n)) + From fd38eb43126af1afbc91630e0a9873b8fce4d311 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:40:44 +0200 Subject: [PATCH 04/55] Update templates --- .../configuration/context.py-template | 13 +++++++++++++ .../configuration/settings.py-template | 9 +-------- .../data/data_providers.py-template | 5 ----- .../{data => data_providers}/__init__.py-template | 0 .../data_providers/data_providers.py-template | 15 +++++++++++++++ .../{strategy => strategies}/__init__.py-template | 0 6 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template delete mode 100755 investing_bot_framework/templates/bot_project_directory/bot_project_template/data/data_providers.py-template rename investing_bot_framework/templates/bot_project_directory/bot_project_template/{data => data_providers}/__init__.py-template (100%) create mode 100755 investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template rename investing_bot_framework/templates/bot_project_directory/bot_project_template/{strategy => strategies}/__init__.py-template (100%) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template new file mode 100644 index 00000000..85b4ad6f --- /dev/null +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template @@ -0,0 +1,13 @@ +from investing_bot_framework.core.context import BotContext +from investing_bot_framework.core.context.states.setup_state import SetupState +from investing_bot_framework.core.context.states.data_provider_state import DataProviderState + +from bot.data_providers.data_providers import MyDataProvider + +# Register Initial state +context = BotContext() +context.register_initial_state(SetupState) + +# Register all data providers +DataProviderState.register_data_providers([MyDataProvider()]) + diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template index e0968492..78fa894c 100755 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template @@ -1,10 +1,3 @@ BOT_PROJECT_NAME = '{{ bot_project_name }}' -INSTALLED_DATA_PROVIDER_APPS = [ - '{{ bot_project_name }}.data', -] - -INSTALLED_STRATEGY_APPS = [ - '{{ bot_project_name }}.strategy', -] - +BOT_CONTEXT_CONFIGURATION = '{{ bot_project_name }}.configuration.context' diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/data/data_providers.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/data/data_providers.py-template deleted file mode 100755 index b18f9061..00000000 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/data/data_providers.py-template +++ /dev/null @@ -1,5 +0,0 @@ -from investing_bot_framework.core.data.data_providers import DataProvider - -""" -Define here all you data providers, e.g. Rest API client -""" diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/data/__init__.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/__init__.py-template similarity index 100% rename from investing_bot_framework/templates/bot_project_directory/bot_project_template/data/__init__.py-template rename to investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/__init__.py-template diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template new file mode 100755 index 00000000..a9765c0b --- /dev/null +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template @@ -0,0 +1,15 @@ +from investing_bot_framework.core.data_providers import DataProvider +from investing_bot_framework.core.utils import TimeUnit + +""" +Define here all you data providers, e.g. Rest API client +""" + + +class MyDataProvider(DataProvider): + time_unit = TimeUnit.SECOND + time_interval = 1 + id = 'my_data_provider' + + def provide_data(self) -> None: + pass diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategy/__init__.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/__init__.py-template similarity index 100% rename from investing_bot_framework/templates/bot_project_directory/bot_project_template/strategy/__init__.py-template rename to investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/__init__.py-template From 126af4f88af361d4f6c5f35a2ba800e7b31e7d49 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:46:28 +0200 Subject: [PATCH 05/55] Remove unused data provider attributes --- .../core/data_providers/__init__.py | 1 + .../core/data_providers/data_provider.py | 55 +++++++++++++++++++ .../core/data_providers/mixins/__init__.py | 1 + .../data_providers/mixins/rest_api_mixin.py | 0 4 files changed, 57 insertions(+) create mode 100644 investing_bot_framework/core/data_providers/__init__.py create mode 100644 investing_bot_framework/core/data_providers/data_provider.py create mode 100644 investing_bot_framework/core/data_providers/mixins/__init__.py rename investing_bot_framework/core/{data => }/data_providers/mixins/rest_api_mixin.py (100%) diff --git a/investing_bot_framework/core/data_providers/__init__.py b/investing_bot_framework/core/data_providers/__init__.py new file mode 100644 index 00000000..1090a6f3 --- /dev/null +++ b/investing_bot_framework/core/data_providers/__init__.py @@ -0,0 +1 @@ +from investing_bot_framework.core.data_providers.data_provider import DataProvider diff --git a/investing_bot_framework/core/data_providers/data_provider.py b/investing_bot_framework/core/data_providers/data_provider.py new file mode 100644 index 00000000..657916b8 --- /dev/null +++ b/investing_bot_framework/core/data_providers/data_provider.py @@ -0,0 +1,55 @@ +from typing import Dict, Any +from abc import abstractmethod + +from investing_bot_framework.core.workers import Worker +from investing_bot_framework.core.utils import TimeUnit + + +class DataProviderException(Exception): + """ + Should be raised when an data_provider related error occurs, for example if an authorization for an API fails, + i.e.: raise DataProviderException('Provided api token is false') + """ + + def __init__(self, message: str) -> None: + super().__init__(self) + self.message = message + + def __str__(self) -> str: + return self.message + + +class DataProvider(Worker): + """ + Class DataProvider: An entity which responsibility is to provide data_providers from an external data_providers + source. Where a data_providers source is defined as any third party service that provides data_providers, + e.g cloud storage, REST API, or website. + + A data_providers provider must always be run with the start function from it´s super class. Otherwise depend + observers will not be updated. + """ + + def get_time_unit(self) -> TimeUnit: + assert getattr(self, 'time_unit', None) is not None, ( + "{} should either include a time_unit attribute, or override the " + "`get_time_unit()`, method.".format(self.__class__.__name__) + ) + + return getattr(self, 'time_unit') + + def get_time_interval(self) -> int: + assert getattr(self, 'time_interval', None) is not None, ( + "{} should either include a time_interval attribute, or override the " + "`get_time_interval()`, method.".format(self.__class__.__name__) + ) + + return getattr(self, 'time_interval') + + @abstractmethod + def provide_data(self) -> None: + pass + + def work(self, **kwargs: Dict[str, Any]) -> None: + self.provide_data() + + diff --git a/investing_bot_framework/core/data_providers/mixins/__init__.py b/investing_bot_framework/core/data_providers/mixins/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/investing_bot_framework/core/data_providers/mixins/__init__.py @@ -0,0 +1 @@ + diff --git a/investing_bot_framework/core/data/data_providers/mixins/rest_api_mixin.py b/investing_bot_framework/core/data_providers/mixins/rest_api_mixin.py similarity index 100% rename from investing_bot_framework/core/data/data_providers/mixins/rest_api_mixin.py rename to investing_bot_framework/core/data_providers/mixins/rest_api_mixin.py From 5259732975bf2d178e41b305ac86a71ec0bdb241 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:47:13 +0200 Subject: [PATCH 06/55] Fix data provider state --- .../core/context/bot_context.py | 4 +- .../core/context/state_validator.py | 4 +- .../context/states/data_provider_state.py | 131 +++++++++++++++ .../core/context/states/data_state.py | 158 ------------------ .../core/context/states/setup_state.py | 53 +----- .../core/context/states/strategy_state.py | 4 +- 6 files changed, 142 insertions(+), 212 deletions(-) create mode 100644 investing_bot_framework/core/context/states/data_provider_state.py delete mode 100644 investing_bot_framework/core/context/states/data_state.py diff --git a/investing_bot_framework/core/context/bot_context.py b/investing_bot_framework/core/context/bot_context.py index d429fea9..96f350b6 100644 --- a/investing_bot_framework/core/context/bot_context.py +++ b/investing_bot_framework/core/context/bot_context.py @@ -18,7 +18,7 @@ class BotContext(metaclass=Singleton): # Settings reference settings = settings - def initialize(self, bot_state: Type[BotState]) -> None: + def register_initial_state(self, bot_state: Type[BotState]) -> None: # Stop the current state of the investing_bot_framework if self._state: @@ -42,7 +42,7 @@ def _check_state(self, raise_exception: bool = False) -> bool: if raise_exception: raise OperationalException( - "Bot context doesn't have a state, Make sure that you set the state of bot either " + "Bot context doesn't have a state. Make sure that you set the state of bot either " "by initializing it or making sure that you transition to a new valid state." ) else: diff --git a/investing_bot_framework/core/context/state_validator.py b/investing_bot_framework/core/context/state_validator.py index 0316ba93..8c111ede 100644 --- a/investing_bot_framework/core/context/state_validator.py +++ b/investing_bot_framework/core/context/state_validator.py @@ -4,8 +4,8 @@ class StateValidator(ABC): """ Class StateValidator: validates the given state. Use this class to change the transition process of a state. - Use it as a hook to decide if a state must transition, e.g. only change to a strategy state if all the - provided data meets a certain threshold. + Use it as a hook to decide if a state must transition, e.g. only change to a strategies state if all the + provided data_providers meets a certain threshold. """ @abstractmethod diff --git a/investing_bot_framework/core/context/states/data_provider_state.py b/investing_bot_framework/core/context/states/data_provider_state.py new file mode 100644 index 00000000..b2c7fe80 --- /dev/null +++ b/investing_bot_framework/core/context/states/data_provider_state.py @@ -0,0 +1,131 @@ +import time +from typing import List +from wrapt import synchronized + +from investing_bot_framework.core.events import Observer +from investing_bot_framework.core.context.bot_context import BotContext +from investing_bot_framework.core.exceptions import OperationalException +from investing_bot_framework.core.context.states import BotState +from investing_bot_framework.core.executors import ExecutionScheduler +from investing_bot_framework.core.data_providers import DataProvider +from investing_bot_framework.core.executors.data_provider_executor import DataProviderExecutor +from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS, SETTINGS_MAX_WORKERS + + +class DataProviderScheduler(ExecutionScheduler): + """ + Data Provider scheduler that will function as a scheduler to make sure it keeps it state across multiple states, + it is defined as a Singleton. + """ + + def __init__(self): + self._configured = False + super(DataProviderScheduler, self).__init__() + + def configure(self, data_providers: List[DataProvider]) -> None: + self._planning = {} + + for data_provider in data_providers: + self.add_execution_task( + execution_id=data_provider.get_id(), + time_unit=data_provider.get_time_unit(), + interval=data_provider.get_time_interval() + ) + + self._configured = True + + @property + def configured(self) -> bool: + return self._configured + + +class DataProviderState(BotState, Observer): + """ + Represent the data_providers state of a bot. This state will load all the defined data_providers providers and will + run them. + + If you want to validate the state before transitioning, provide a state validator. + """ + + registered_data_providers: List = None + + from investing_bot_framework.core.context.states.strategy_state import StrategyState + transition_state_class = StrategyState + + data_provider_scheduler: DataProviderScheduler = None + + def __init__(self, context: BotContext) -> None: + super(DataProviderState, self).__init__(context) + self._updated = False + self.data_provider_executor = None + + def _schedule_data_providers(self) -> List[DataProvider]: + + if not DataProviderState.data_provider_scheduler: + DataProviderState.data_provider_scheduler = DataProviderScheduler() + + if not DataProviderState.data_provider_scheduler.configured: + DataProviderState.data_provider_scheduler.configure(self.registered_data_providers) + + planning = DataProviderState.data_provider_scheduler.schedule_executions() + planned_data_providers = [] + + for data_provider in self.registered_data_providers: + + if data_provider.get_id() in planning: + planned_data_providers.append(data_provider) + + return planned_data_providers + + def _start_data_providers(self, data_providers: List[DataProvider]) -> None: + + self.data_provider_executor = DataProviderExecutor( + data_providers=data_providers, + max_workers=self.context.settings.get(SETTINGS_MAX_WORKERS, DEFAULT_MAX_WORKERS) + ) + + if self.data_provider_executor.configured: + self.data_provider_executor.add_observer(self) + self.data_provider_executor.start() + else: + # Skip the execution + self._updated = True + + def run(self) -> None: + + if self.registered_data_providers is None: + raise OperationalException("Data providing state has not any data providers configured") + + # Schedule the data_providers providers + planned_data_providers = self._schedule_data_providers() + + # Execute all the data_providers providers + self._start_data_providers(planned_data_providers) + + # Sleep till updated + while not self._updated: + time.sleep(1) + + # Collect all data_providers from the data_providers providers + for data_provider in self.data_provider_executor.registered_data_providers: + print("Data provider: {} finished running".format(data_provider.get_id())) + + def stop(self) -> None: + """ + Stop all data_providers providers + """ + pass + + @synchronized + def update(self, observable, **kwargs) -> None: + self._updated = True + + @staticmethod + def register_data_providers(data_providers: List) -> None: + DataProviderState.registered_data_providers = data_providers + + def reconfigure(self) -> None: + pass + + + diff --git a/investing_bot_framework/core/context/states/data_state.py b/investing_bot_framework/core/context/states/data_state.py deleted file mode 100644 index d4d5192a..00000000 --- a/investing_bot_framework/core/context/states/data_state.py +++ /dev/null @@ -1,158 +0,0 @@ -import time -from typing import List -from wrapt import synchronized - -from investing_bot_framework.core.exceptions import ImproperlyConfigured, OperationalException -from investing_bot_framework.core.events import Observer -from investing_bot_framework.core.context.bot_context import BotContext -from investing_bot_framework.core.context.states import BotState -from investing_bot_framework.core.executors import ExecutionScheduler -from investing_bot_framework.core.data.data_providers import DataProvider, DataProviderExecutor -from investing_bot_framework.core.utils import Singleton, TimeUnit -from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS, SETTINGS_MAX_WORKERS, \ - SETTINGS_DATA_PROVIDER_REGISTERED_APPS - - -def import_class(cl): - d = cl.rfind(".") - classname = cl[d+1:len(cl)] - m = __import__(cl[0:d], globals(), locals(), [classname]) - return getattr(m, classname) - - -class DataProviderScheduler(ExecutionScheduler, metaclass=Singleton): - """ - Data Provider scheduler that will function as a scheduler to make sure it keeps it state across multiple states, - it is defined as a Singleton. - """ - - def __init__(self): - self._configured = False - - super(DataProviderScheduler, self).__init__() - - def configure(self, data_providers: List[DataProvider]) -> None: - self._planning = {} - - for data_provider in data_providers: - self.add_execution_task(execution_id=data_provider.get_id(), time_unit=TimeUnit.ALWAYS) - - self._configured = True - - @property - def configured(self) -> bool: - return self._configured - - -class DataState(BotState, Observer): - """ - Represent the data state of a bot. This state will load all the defined data providers and will - run them. - - If you want to validate the state before transitioning, provide a state validator. - """ - - from investing_bot_framework.core.context.states.strategy_state import StrategyState - transition_state_class = StrategyState - - data_providers: List[DataProvider] = [] - _data_provider_executor: DataProviderExecutor - - def __init__(self, context: BotContext) -> None: - super(DataState, self).__init__(context) - - self._updated = False - self._configured = False - self._initialize() - - def _initialize(self) -> None: - """ - Initializes the data providers, loads them dynamically from the specified settings - """ - self._clean_up() - - for data_provider_app_class in self.context.settings[SETTINGS_DATA_PROVIDER_REGISTERED_APPS]: - - instance = import_class(data_provider_app_class)() - - if not isinstance(instance, DataProvider): - raise ImproperlyConfigured( - "Specified data provider {} is not a instance of DataProvider".format(data_provider_app_class) - ) - - self.data_providers.append(instance) - - self._configured = True - - def _clean_up(self) -> None: - """ - Cleans up all the data state resources - """ - - self._data_provider_executor = None - self.data_providers = [] - - def _schedule_data_providers(self) -> List[DataProvider]: - data_provider_scheduler = DataProviderScheduler() - - if not data_provider_scheduler.configured: - data_provider_scheduler.configure(self.data_providers) - - planning = data_provider_scheduler.schedule_executions() - planned_data_providers = [] - - for data_provider in self.data_providers: - - if data_provider.get_id() in planning: - planned_data_providers.append(data_provider) - - return planned_data_providers - - def _start_data_providers(self, data_providers: List[DataProvider]) -> None: - - self._data_provider_executor = DataProviderExecutor( - data_providers=data_providers, - max_workers=self.context.settings.get(SETTINGS_MAX_WORKERS, DEFAULT_MAX_WORKERS) - ) - - self._data_provider_executor.add_observer(self) - self._data_provider_executor.start() - - def run(self) -> None: - - if self._configured: - # Schedule the data providers - planned_data_providers = self._schedule_data_providers() - - # Execute all the data providers - self._start_data_providers(planned_data_providers) - - # Sleep till updated - while not self._updated: - time.sleep(1) - - # Collect all data from the data providers - for data_provider in self._data_provider_executor.registered_data_providers: - print("Data provider: {} finished running".format(data_provider.get_id())) - - else: - raise OperationalException("Data state started without any configuration") - - def stop(self) -> None: - """ - Stop all data providers - """ - - if self._configured: - - if self._data_provider_executor.processing: - self._data_provider_executor.stop() - - def reconfigure(self) -> None: - self._clean_up() - self._initialize() - - @synchronized - def update(self, observable, **kwargs) -> None: - self._updated = True - diff --git a/investing_bot_framework/core/context/states/setup_state.py b/investing_bot_framework/core/context/states/setup_state.py index 9e6ce13a..6dc1061f 100644 --- a/investing_bot_framework/core/context/states/setup_state.py +++ b/investing_bot_framework/core/context/states/setup_state.py @@ -1,16 +1,11 @@ -from pydoc import locate - from investing_bot_framework.core.exceptions import ImproperlyConfigured from investing_bot_framework.core.context.states import BotState -from investing_bot_framework.core.resolvers import ClassCollector -from investing_bot_framework.core.data.data_providers import DataProvider -from investing_bot_framework.core.configuration.config_constants import SETTINGS_DATA_PROVIDER_REGISTERED_APPS class SetupState(BotState): - from investing_bot_framework.core.context.states.data_state import DataState - transition_state_class = DataState + from investing_bot_framework.core.context.states.data_provider_state import DataProviderState + transition_state_class = DataProviderState def __init__(self, context): super(SetupState, self).__init__(context) @@ -27,42 +22,11 @@ def run(self) -> None: # Load the settings if not self.context.settings.configured: raise ImproperlyConfigured( - "Settings module is not specified, make sure you have setup a investing_bot_framework project and the investing_bot_framework is valid or that " - "you have specified the settings module in your manage.py file" - ) - - # Initialize all data provider executors - self._validate_data_providers() - - def _validate_data_providers(self) -> None: - """ - Validates if all the data providers are correctly configured and can be loaded. - """ - - data_provider_apps_config = self.context.settings.get(SETTINGS_DATA_PROVIDER_REGISTERED_APPS, None) - - # Check if any data providers are configured - if data_provider_apps_config is None or len(data_provider_apps_config) < 1: - raise ImproperlyConfigured( - "You have not configured any data provider apps in your settings file. Please define your data " - "provider apps in your settings file. If you have difficulties configuring data providers, consider " - "looking at the documentation." + "Settings module is not specified, make sure you have setup a investing_bot_framework project and " + "the investing_bot_framework is valid or that you have specified the settings module in your " + "manage.py file" ) - # Try to load all the specified data provider modules - for data_provider_app in data_provider_apps_config: - instance = locate(data_provider_app) - print(instance) - # class_collector = ClassCollector(package_path=data_provider_app, class_type=DataProvider) - # - # if len(class_collector.instances) == 0: - # raise ImproperlyConfigured( - # "Could not load data providers from package {}, are they implemented correctly?. Please make sure " - # "that you defined the right package or module. In the case of referring to your own defined data " - # "providers make sure that they can be imported. If you have difficulties configuring data " - # "providers, consider looking at the documentation.".format(data_provider_app) - # ) - def stop(self) -> None: # Stopping all services pass @@ -70,10 +34,3 @@ def stop(self) -> None: def reconfigure(self) -> None: # Clean up and reconfigure all the services pass - - def transition(self) -> None: - - # Transition to data state - from investing_bot_framework.core.context.states.data_state import DataState - self.context.transition_to(DataState) - self.context.run() diff --git a/investing_bot_framework/core/context/states/strategy_state.py b/investing_bot_framework/core/context/states/strategy_state.py index 661f14ba..94b6c719 100644 --- a/investing_bot_framework/core/context/states/strategy_state.py +++ b/investing_bot_framework/core/context/states/strategy_state.py @@ -17,5 +17,5 @@ def update(self, observable, **kwargs) -> None: pass def get_transition_state_class(self) -> Type: - from investing_bot_framework.core.context.states.data_state import DataState - return DataState + from investing_bot_framework.core.context.states.data_provider_state import DataProviderState + return DataProviderState From 8d8d1369f910951b486180edaeb1be33e1c27bf7 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:47:37 +0200 Subject: [PATCH 07/55] Add context configuration --- investing_bot_framework/core/configuration/config_constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/investing_bot_framework/core/configuration/config_constants.py b/investing_bot_framework/core/configuration/config_constants.py index edd0edcb..62a9ee35 100644 --- a/investing_bot_framework/core/configuration/config_constants.py +++ b/investing_bot_framework/core/configuration/config_constants.py @@ -8,6 +8,7 @@ SETTINGS_DATA_PROVIDER_REGISTERED_APPS = 'INSTALLED_DATA_PROVIDER_APPS' SETTINGS_STRATEGY_REGISTERED_APPS = 'INSTALLED_STRATEGY_APPS' SETTINGS_MAX_WORKERS = 'DEFAULT_MAX_WORKERS' +BOT_CONTEXT_CONFIGURATION = 'BOT_CONTEXT_CONFIGURATION' # Operational constants DEFAULT_MAX_WORKERS = 2 From d2bcb51ba7c77c11108499f245b7941688b5cd37 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:48:10 +0200 Subject: [PATCH 08/55] Rearange data provider files --- investing_bot_framework/core/data/__init__.py | 0 .../core/data/data_providers/__init__.py | 2 - .../core/data/data_providers/data_provider.py | 55 ------------------- .../data/data_providers/mixins/__init__.py | 1 - 4 files changed, 58 deletions(-) delete mode 100644 investing_bot_framework/core/data/__init__.py delete mode 100644 investing_bot_framework/core/data/data_providers/__init__.py delete mode 100644 investing_bot_framework/core/data/data_providers/data_provider.py delete mode 100644 investing_bot_framework/core/data/data_providers/mixins/__init__.py diff --git a/investing_bot_framework/core/data/__init__.py b/investing_bot_framework/core/data/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/investing_bot_framework/core/data/data_providers/__init__.py b/investing_bot_framework/core/data/data_providers/__init__.py deleted file mode 100644 index ddad2889..00000000 --- a/investing_bot_framework/core/data/data_providers/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from investing_bot_framework.core.data.data_providers.data_provider import DataProvider, RestApiDataProvider -from investing_bot_framework.core.data.data_providers.data_provider_executor import DataProviderExecutor diff --git a/investing_bot_framework/core/data/data_providers/data_provider.py b/investing_bot_framework/core/data/data_providers/data_provider.py deleted file mode 100644 index 297412b2..00000000 --- a/investing_bot_framework/core/data/data_providers/data_provider.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Dict, Any -from abc import abstractmethod, ABC - -from investing_bot_framework.core.workers import Worker -from investing_bot_framework.core.data.data_providers.mixins import RestApiClientMixin - - -class DataProviderException(Exception): - """ - Should be raised when an data_provider related error occurs, for example if an authorization for an API fails, - i.e.: raise DataProviderException('Provided api token is false') - """ - - def __init__(self, message: str) -> None: - super().__init__(self) - self.message = message - - def __str__(self) -> str: - return self.message - - -class DataProvider(Worker): - """ - Class DataProvider: An entity which responsibility is to provide data from an external data source. Where a data - source is defined as any third party service that provides data, e.g cloud storage, REST API, or website. - - A data provider must always be run with the start function from it´s super class. Otherwise depend observers will - not be updated. - """ - - def __init__(self): - super(DataProvider, self).__init__() - self._data: Any = None - - @abstractmethod - def provide_data(self, **kwargs: Dict[str, Any]) -> Any: - pass - - def work(self, **kwargs: Dict[str, Any]) -> None: - self._data = self.provide_data() - - @property - def data(self) -> Any: - - if self._data is None: - raise DataProviderException("Could not provide data, data is not set by {}".format(self.get_id())) - - return self._data - - def clean_up(self) -> None: - self._data = None - - -class RestApiDataProvider(RestApiClientMixin, DataProvider, ABC): - pass diff --git a/investing_bot_framework/core/data/data_providers/mixins/__init__.py b/investing_bot_framework/core/data/data_providers/mixins/__init__.py deleted file mode 100644 index d459f32a..00000000 --- a/investing_bot_framework/core/data/data_providers/mixins/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from investing_bot_framework.core.data.data_providers.mixins.rest_api_mixin import RestApiClientMixin From 214eec69dff27a31bec97006fae3ae1df1bc1c5e Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:50:55 +0200 Subject: [PATCH 09/55] Add import module --- .../core/management/commands/runbot.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/investing_bot_framework/core/management/commands/runbot.py b/investing_bot_framework/core/management/commands/runbot.py index bebe4389..6d570456 100644 --- a/investing_bot_framework/core/management/commands/runbot.py +++ b/investing_bot_framework/core/management/commands/runbot.py @@ -2,13 +2,14 @@ from investing_bot_framework.core.management.command import BaseCommand, CommandError from investing_bot_framework.core.context import BotContext -from investing_bot_framework.core.context.states.setup_state import SetupState +from investing_bot_framework.core.configuration.config_constants import BOT_CONTEXT_CONFIGURATION +from investing_bot_framework.core.configuration import settings class RunBotCommand(BaseCommand): help_message = ( - "Runs a investing_bot_framework, by default it will run until stopped, if cycles is specified it will run the according to " - "the amount of cycles" + "Runs a investing_bot_framework, by default it will run until stopped, if cycles is specified it will run " + "the according to the amount of cycles" ) success_message = ( @@ -23,11 +24,16 @@ def add_arguments(self, parser) -> None: def handle(self, *args, **options) -> Any: + # configure settings + settings.configure() + + # Load the context configuration + __import__(settings[BOT_CONTEXT_CONFIGURATION]) + cycles = options.get('name', None) # Create an investing_bot_framework context of the investing_bot_framework and run it context = BotContext() - context.initialize(SetupState) context.start() @staticmethod From d6c04367136d36e8b424e3c89f9ac9ea402aa742 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:55:07 +0200 Subject: [PATCH 10/55] Add import module --- .../core/resolvers/module_loaders/data_providers_loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/investing_bot_framework/core/resolvers/module_loaders/data_providers_loader.py b/investing_bot_framework/core/resolvers/module_loaders/data_providers_loader.py index 58eafcf9..84e79d20 100644 --- a/investing_bot_framework/core/resolvers/module_loaders/data_providers_loader.py +++ b/investing_bot_framework/core/resolvers/module_loaders/data_providers_loader.py @@ -3,7 +3,7 @@ from investing_bot_framework.core.exceptions import ImproperlyConfigured from investing_bot_framework.core.configuration import settings from investing_bot_framework.core.resolvers import ClassCollector -from investing_bot_framework.core.data.data_providers import DataProvider +from investing_bot_framework.core.data_providers import DataProvider class DataProvidersLoader: @@ -28,7 +28,8 @@ def load_modules(self) -> List[DataProvider]: if len(class_collector.instances) == 0: raise ImproperlyConfigured( - "There are no data providers configured. Make sure you implement data providers or use a plugin" + "There are no data_providers providers configured. Make sure you implement data_providers providers " + "or use a plugin" ) return class_collector.instances From 522ca0552995c7707010411b572f0f2ae0fffb3e Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:55:41 +0200 Subject: [PATCH 11/55] Fix imports --- .../data_provider_executor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) rename investing_bot_framework/core/{data/data_providers => executors}/data_provider_executor.py (76%) diff --git a/investing_bot_framework/core/data/data_providers/data_provider_executor.py b/investing_bot_framework/core/executors/data_provider_executor.py similarity index 76% rename from investing_bot_framework/core/data/data_providers/data_provider_executor.py rename to investing_bot_framework/core/executors/data_provider_executor.py index 919a4bde..8da171de 100644 --- a/investing_bot_framework/core/data/data_providers/data_provider_executor.py +++ b/investing_bot_framework/core/executors/data_provider_executor.py @@ -1,7 +1,7 @@ from typing import List from investing_bot_framework.core.workers import Worker -from investing_bot_framework.core.data.data_providers import DataProvider +from investing_bot_framework.core.data_providers import DataProvider from investing_bot_framework.core.executors import Executor from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS @@ -16,7 +16,7 @@ def __init__(self, data_providers: List[DataProvider] = None, max_workers: int = self._registered_data_providers: List[DataProvider] = [] - if data_providers is not None: + if data_providers is not None and len(data_providers) > 0: self._registered_data_providers = data_providers def create_workers(self) -> List[Worker]: @@ -25,3 +25,7 @@ def create_workers(self) -> List[Worker]: @property def registered_data_providers(self) -> List[DataProvider]: return self._registered_data_providers + + @property + def configured(self): + return self._registered_data_providers is not None and len(self._registered_data_providers) > 0 \ No newline at end of file From b32300c53d7e997728dfcee746af80ffb9331135 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:56:37 +0200 Subject: [PATCH 12/55] Update tests --- .../tests/core/data/data_providers/data_provider.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/investing_bot_framework/tests/core/data/data_providers/data_provider.py b/investing_bot_framework/tests/core/data/data_providers/data_provider.py index cef1059a..28dac426 100644 --- a/investing_bot_framework/tests/core/data/data_providers/data_provider.py +++ b/investing_bot_framework/tests/core/data/data_providers/data_provider.py @@ -1,7 +1,7 @@ from typing import Dict, Any from unittest import TestCase -from investing_bot_framework.core.data.data_providers import DataProvider +from investing_bot_framework.core.data_providers import DataProvider from investing_bot_framework.core.events import Observer @@ -10,7 +10,7 @@ class TestDataProviderOne(DataProvider): id = 'TestDataProviderOne' def provide_data(self, **kwargs: Dict[str, Any]) -> Any: - return "data" + return "data_providers" class TestDataProviderTwo(DataProvider): @@ -18,7 +18,7 @@ class TestDataProviderTwo(DataProvider): id = 'TestDataProviderTwo' def provide_data(self, **kwargs: Dict[str, Any]) -> Any: - return "data" + return "data_providers" class TestObserver(Observer): @@ -41,7 +41,7 @@ def test(self): observer = TestObserver() data_provider_one.add_observer(observer) - # Run the data provider + # Run the data_providers provider data_provider_one.start() # Observer must have been updated @@ -54,7 +54,7 @@ def test(self): data_provider_two.add_observer(observer) - # Run the data provider + # Run the data_providers provider data_provider_two.start() # Observer must have been updated From 700e19a35518cfc220d31702b1f537d50d889531 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 12:59:38 +0200 Subject: [PATCH 13/55] Add logging configuration --- .../configuration/settings.py-template | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template index 78fa894c..22787869 100755 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template @@ -1,3 +1,25 @@ +import logging +from pathlib import Path + BOT_PROJECT_NAME = '{{ bot_project_name }}' BOT_CONTEXT_CONFIGURATION = '{{ bot_project_name }}.configuration.context' + +BASE_DIR = str(Path(__file__).parent) + +LOG_PATH = BASE_DIR + +LOG_FILE_NAME = 'log' + +# Logger configuration +logging.basicConfig(level=logging.INFO) +logFormatter = logging.Formatter("%(asctime)s [%(threadName)] [%(levelname)] %(message)s") +rootLogger = logging.getLogger() + +fileHandler = logging.FileHandler("{0}/{1}.log".format(LOG_PATH, LOG_FILE_NAME)) +fileHandler.setFormatter(logFormatter) +rootLogger.addHandler(fileHandler) + +consoleHandler = logging.StreamHandler() +consoleHandler.setFormatter(logFormatter) +rootLogger.addHandler(consoleHandler) \ No newline at end of file From bda6d5fad6d6647c7a25391689f7b73e66e6d750 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:01:13 +0200 Subject: [PATCH 14/55] Add logger --- investing_bot_framework/core/extensions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/investing_bot_framework/core/extensions.py b/investing_bot_framework/core/extensions.py index a96f80c7..22f62884 100644 --- a/investing_bot_framework/core/extensions.py +++ b/investing_bot_framework/core/extensions.py @@ -1,3 +1,7 @@ +import logging + from investing_bot_framework.core.resolvers import DatabaseResolver db = DatabaseResolver() + +logger = logging.getLogger('investing-bot-framework') From 6de9566b35c067beb5250f98aae92aef21be2ebf Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:37:53 +0200 Subject: [PATCH 15/55] Change config constants to SETTINGS_BOT_CONTEXT_CONFIGURATION --- investing_bot_framework/core/management/commands/runbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/investing_bot_framework/core/management/commands/runbot.py b/investing_bot_framework/core/management/commands/runbot.py index 6d570456..0b45964b 100644 --- a/investing_bot_framework/core/management/commands/runbot.py +++ b/investing_bot_framework/core/management/commands/runbot.py @@ -2,7 +2,7 @@ from investing_bot_framework.core.management.command import BaseCommand, CommandError from investing_bot_framework.core.context import BotContext -from investing_bot_framework.core.configuration.config_constants import BOT_CONTEXT_CONFIGURATION +from investing_bot_framework.core.configuration.config_constants import SETTINGS_BOT_CONTEXT_CONFIGURATION from investing_bot_framework.core.configuration import settings @@ -28,7 +28,7 @@ def handle(self, *args, **options) -> Any: settings.configure() # Load the context configuration - __import__(settings[BOT_CONTEXT_CONFIGURATION]) + __import__(settings[SETTINGS_BOT_CONTEXT_CONFIGURATION]) cycles = options.get('name', None) From 313bc8cbb17570e2d213df1819abe27ed6524181 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:45:38 +0200 Subject: [PATCH 16/55] Remove logger declaration --- investing_bot_framework/core/extensions.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/investing_bot_framework/core/extensions.py b/investing_bot_framework/core/extensions.py index 22f62884..60881642 100644 --- a/investing_bot_framework/core/extensions.py +++ b/investing_bot_framework/core/extensions.py @@ -1,7 +1,3 @@ -import logging - from investing_bot_framework.core.resolvers import DatabaseResolver -db = DatabaseResolver() - -logger = logging.getLogger('investing-bot-framework') +db = DatabaseResolver() \ No newline at end of file From 09cd2c2428b0144b9696a5b91aae41ac3ae443fe Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:46:15 +0200 Subject: [PATCH 17/55] Setup logging config --- investing_bot_framework/core/configuration/__init__.py | 5 ++++- .../core/configuration/config_constants.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/investing_bot_framework/core/configuration/__init__.py b/investing_bot_framework/core/configuration/__init__.py index 7efb3279..562ffdc5 100644 --- a/investing_bot_framework/core/configuration/__init__.py +++ b/investing_bot_framework/core/configuration/__init__.py @@ -1,4 +1,5 @@ import os +import logging.config from typing import Any from importlib import import_module from enum import Enum @@ -6,7 +7,7 @@ from investing_bot_framework.core.exceptions import ImproperlyConfigured, OperationalException from investing_bot_framework.core.configuration.template import Template from investing_bot_framework.core.configuration.config_constants import SETTINGS_MODULE_PATH_ENV_NAME, \ - SETTINGS_STRATEGY_REGISTERED_APPS, SETTINGS_DATA_PROVIDER_REGISTERED_APPS, BASE_DIR + SETTINGS_STRATEGY_REGISTERED_APPS, SETTINGS_DATA_PROVIDER_REGISTERED_APPS, BASE_DIR, SETTINGS_LOGGING_CONFIG class TimeUnit(Enum): @@ -96,6 +97,8 @@ def configure(self, settings_module: str = None) -> None: self._configured = True + logging.config.dictConfig(self[SETTINGS_LOGGING_CONFIG]) + @property def settings_module(self) -> str: return self._settings_module diff --git a/investing_bot_framework/core/configuration/config_constants.py b/investing_bot_framework/core/configuration/config_constants.py index 62a9ee35..de89dd8f 100644 --- a/investing_bot_framework/core/configuration/config_constants.py +++ b/investing_bot_framework/core/configuration/config_constants.py @@ -8,7 +8,8 @@ SETTINGS_DATA_PROVIDER_REGISTERED_APPS = 'INSTALLED_DATA_PROVIDER_APPS' SETTINGS_STRATEGY_REGISTERED_APPS = 'INSTALLED_STRATEGY_APPS' SETTINGS_MAX_WORKERS = 'DEFAULT_MAX_WORKERS' -BOT_CONTEXT_CONFIGURATION = 'BOT_CONTEXT_CONFIGURATION' +SETTINGS_BOT_CONTEXT_CONFIGURATION = 'BOT_CONTEXT_CONFIGURATION' +SETTINGS_LOGGING_CONFIG = 'LOGGING' # Operational constants DEFAULT_MAX_WORKERS = 2 From 5affedbfe3817c4992c797991704753ce3d9dcd6 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:47:44 +0200 Subject: [PATCH 18/55] Add logging --- investing_bot_framework/core/data_providers/data_provider.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/investing_bot_framework/core/data_providers/data_provider.py b/investing_bot_framework/core/data_providers/data_provider.py index 657916b8..f179f146 100644 --- a/investing_bot_framework/core/data_providers/data_provider.py +++ b/investing_bot_framework/core/data_providers/data_provider.py @@ -1,9 +1,12 @@ +import logging from typing import Dict, Any from abc import abstractmethod from investing_bot_framework.core.workers import Worker from investing_bot_framework.core.utils import TimeUnit +logger = logging.getLogger(__name__) + class DataProviderException(Exception): """ @@ -50,6 +53,7 @@ def provide_data(self) -> None: pass def work(self, **kwargs: Dict[str, Any]) -> None: + logger.info("Starting data provider {}".format(self.get_id())) self.provide_data() From d28e60b590a2b8bf70560fb0e81ae70dd85aa094 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:49:57 +0200 Subject: [PATCH 19/55] Add settings configuration for logging --- .../configuration/settings.py-template | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template index 22787869..7df34664 100755 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template @@ -1,25 +1,51 @@ -import logging +import os from pathlib import Path -BOT_PROJECT_NAME = '{{ bot_project_name }}' +BOT_PROJECT_NAME = 'bot' -BOT_CONTEXT_CONFIGURATION = '{{ bot_project_name }}.configuration.context' +BOT_CONTEXT_CONFIGURATION = 'bot.configuration.context' -BASE_DIR = str(Path(__file__).parent) +# Change this when not in development, feature or hot-fix branch +DEBUG = int(os.environ.get('DEBUG', True)) -LOG_PATH = BASE_DIR +BASE_DIR = str(Path(__file__).parent.parent) LOG_FILE_NAME = 'log' -# Logger configuration -logging.basicConfig(level=logging.INFO) -logFormatter = logging.Formatter("%(asctime)s [%(threadName)] [%(levelname)] %(message)s") -rootLogger = logging.getLogger() - -fileHandler = logging.FileHandler("{0}/{1}.log".format(LOG_PATH, LOG_FILE_NAME)) -fileHandler.setFormatter(logFormatter) -rootLogger.addHandler(fileHandler) - -consoleHandler = logging.StreamHandler() -consoleHandler.setFormatter(logFormatter) -rootLogger.addHandler(consoleHandler) \ No newline at end of file +LOG_PATH = "{}{}.log".format(BASE_DIR, LOG_FILE_NAME) + +if DEBUG: + logging_level = "DEBUG" +else: + logging_level = "INFO" + +# Logging configuration +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(levelname)s %(asctime)s - [thread: %(threadName)-4s %(name)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'standard', + }, + 'file': { + 'formatter': 'standard', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': LOG_PATH, + 'backupCount': 10, + 'maxBytes': 10000, + }, + }, + 'loggers': { + '': { + 'level': logging_level, + 'handlers': ['console', 'file'], + }, + }, +} \ No newline at end of file From 4a7a6dc3e50a836666e971df05f339bb0ffcd38b Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 13:50:23 +0200 Subject: [PATCH 20/55] Remove unused settings --- investing_bot_framework/settings.py | 54 ----------------------------- 1 file changed, 54 deletions(-) delete mode 100644 investing_bot_framework/settings.py diff --git a/investing_bot_framework/settings.py b/investing_bot_framework/settings.py deleted file mode 100644 index 4a3d1cb0..00000000 --- a/investing_bot_framework/settings.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import logging.config - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - -# Change this when not in development, feature or hot-fix branch -DEBUG = int(os.environ.get('DEBUG', True)) - -# Setup logging -# make sure that the log dir exists -log_dir = os.path.abspath(os.path.join(BASE_DIR, 'logs')) -main_log_file = os.path.join(log_dir, 'main.log') - -if not os.path.isdir(log_dir): - os.mkdir(log_dir) - -if not os.path.isfile(main_log_file): - os.mknod(main_log_file) - -if DEBUG: - logging_level = "DEBUG" -else: - logging_level = "INFO" - -logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'standard': { - 'format': '%(levelname)s %(asctime)s - [thread: %(threadName)-4s %(name)s] %(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'standard', - }, - 'file': { - 'formatter': 'standard', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': os.path.join(BASE_DIR, 'logs/main.log'), - 'backupCount': 10, - 'maxBytes': 10000, - }, - }, - 'loggers': { - '': { - 'level': logging_level, - 'handlers': ['console', 'file'], - }, - }, -}) From 987e63f823330f7aa5db6f7e8032befaa4073131 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 22:27:59 +0200 Subject: [PATCH 21/55] Setup logging --- investing_bot_framework/core/context/bot_context.py | 1 - .../core/context/states/data_provider_state.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/investing_bot_framework/core/context/bot_context.py b/investing_bot_framework/core/context/bot_context.py index 96f350b6..96eb5c1c 100644 --- a/investing_bot_framework/core/context/bot_context.py +++ b/investing_bot_framework/core/context/bot_context.py @@ -30,7 +30,6 @@ def transition_to(self, bot_state: Type[BotState]) -> None: """ Function to change the running BotState at runtime. """ - self._state = bot_state(context=self) def _check_state(self, raise_exception: bool = False) -> bool: diff --git a/investing_bot_framework/core/context/states/data_provider_state.py b/investing_bot_framework/core/context/states/data_provider_state.py index b2c7fe80..d9d829c4 100644 --- a/investing_bot_framework/core/context/states/data_provider_state.py +++ b/investing_bot_framework/core/context/states/data_provider_state.py @@ -1,4 +1,5 @@ import time +import logging from typing import List from wrapt import synchronized @@ -11,6 +12,8 @@ from investing_bot_framework.core.executors.data_provider_executor import DataProviderExecutor from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS, SETTINGS_MAX_WORKERS +logger = logging.getLogger(__name__) + class DataProviderScheduler(ExecutionScheduler): """ @@ -108,7 +111,7 @@ def run(self) -> None: # Collect all data_providers from the data_providers providers for data_provider in self.data_provider_executor.registered_data_providers: - print("Data provider: {} finished running".format(data_provider.get_id())) + logger.info("Data provider: {} finished running".format(data_provider.get_id())) def stop(self) -> None: """ From 956ac03941aa017888b6e64b8c8069dd35397c13 Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 22:34:22 +0200 Subject: [PATCH 22/55] Initial commit for scheduled worker --- .../core/workers/__init__.py | 2 ++ .../core/workers/scheduled_worker.py | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 investing_bot_framework/core/workers/scheduled_worker.py diff --git a/investing_bot_framework/core/workers/__init__.py b/investing_bot_framework/core/workers/__init__.py index 511deca6..12d5b55b 100644 --- a/investing_bot_framework/core/workers/__init__.py +++ b/investing_bot_framework/core/workers/__init__.py @@ -1 +1,3 @@ from investing_bot_framework.core.workers.worker import Worker +from investing_bot_framework.core.workers.scheduled_worker import ScheduledWorker + diff --git a/investing_bot_framework/core/workers/scheduled_worker.py b/investing_bot_framework/core/workers/scheduled_worker.py new file mode 100644 index 00000000..722bfa45 --- /dev/null +++ b/investing_bot_framework/core/workers/scheduled_worker.py @@ -0,0 +1,23 @@ +from abc import ABC + +from investing_bot_framework.core.utils import TimeUnit +from investing_bot_framework.core.workers.worker import Worker + + +class ScheduledWorker(Worker, ABC): + + def get_time_unit(self) -> TimeUnit: + assert getattr(self, 'time_unit', None) is not None, ( + "{} should either include a time_unit attribute, or override the " + "`get_time_unit()`, method.".format(self.__class__.__name__) + ) + + return getattr(self, 'time_unit') + + def get_time_interval(self) -> int: + assert getattr(self, 'time_interval', None) is not None, ( + "{} should either include a time_interval attribute, or override the " + "`get_time_interval()`, method.".format(self.__class__.__name__) + ) + + return getattr(self, 'time_interval') From 827a2d69e00d695c217e7ea70d8731b0c7368dec Mon Sep 17 00:00:00 2001 From: sadays Date: Sat, 13 Jun 2020 22:35:35 +0200 Subject: [PATCH 23/55] Change worker to ScheduledWorker --- .../core/data_providers/data_provider.py | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/investing_bot_framework/core/data_providers/data_provider.py b/investing_bot_framework/core/data_providers/data_provider.py index f179f146..eab26b75 100644 --- a/investing_bot_framework/core/data_providers/data_provider.py +++ b/investing_bot_framework/core/data_providers/data_provider.py @@ -2,8 +2,7 @@ from typing import Dict, Any from abc import abstractmethod -from investing_bot_framework.core.workers import Worker -from investing_bot_framework.core.utils import TimeUnit +from investing_bot_framework.core.workers import ScheduledWorker logger = logging.getLogger(__name__) @@ -22,7 +21,7 @@ def __str__(self) -> str: return self.message -class DataProvider(Worker): +class DataProvider(ScheduledWorker): """ Class DataProvider: An entity which responsibility is to provide data_providers from an external data_providers source. Where a data_providers source is defined as any third party service that provides data_providers, @@ -32,22 +31,6 @@ class DataProvider(Worker): observers will not be updated. """ - def get_time_unit(self) -> TimeUnit: - assert getattr(self, 'time_unit', None) is not None, ( - "{} should either include a time_unit attribute, or override the " - "`get_time_unit()`, method.".format(self.__class__.__name__) - ) - - return getattr(self, 'time_unit') - - def get_time_interval(self) -> int: - assert getattr(self, 'time_interval', None) is not None, ( - "{} should either include a time_interval attribute, or override the " - "`get_time_interval()`, method.".format(self.__class__.__name__) - ) - - return getattr(self, 'time_interval') - @abstractmethod def provide_data(self) -> None: pass From 68db190a3916643e4fbaaa87ca192e9b12e571cf Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 00:06:38 +0200 Subject: [PATCH 24/55] Initial commit for Strategy --- .../core/strategies/strategy.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 investing_bot_framework/core/strategies/strategy.py diff --git a/investing_bot_framework/core/strategies/strategy.py b/investing_bot_framework/core/strategies/strategy.py new file mode 100644 index 00000000..21fea0ef --- /dev/null +++ b/investing_bot_framework/core/strategies/strategy.py @@ -0,0 +1,23 @@ +import logging +from typing import Dict, Any +from abc import abstractmethod + +from investing_bot_framework.core.workers import ScheduledWorker + +logger = logging.getLogger(__name__) + + +class Strategy(ScheduledWorker): + """ + Class Strategy + """ + + @abstractmethod + def apply_strategy(self) -> None: + pass + + def work(self, **kwargs: Dict[str, Any]) -> None: + logger.info("Starting strategy {}".format(self.get_id())) + self.apply_strategy() + + From 7c2f657f7ef936753ff340b3794a329ce013d4ec Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 00:07:10 +0200 Subject: [PATCH 25/55] Remove data provider executor --- .../core/executors/__init__.py | 2 +- .../core/executors/data_provider_executor.py | 31 ------------------- 2 files changed, 1 insertion(+), 32 deletions(-) delete mode 100644 investing_bot_framework/core/executors/data_provider_executor.py diff --git a/investing_bot_framework/core/executors/__init__.py b/investing_bot_framework/core/executors/__init__.py index 0349763c..93f5ebe1 100644 --- a/investing_bot_framework/core/executors/__init__.py +++ b/investing_bot_framework/core/executors/__init__.py @@ -1,2 +1,2 @@ from investing_bot_framework.core.executors.executor import Executor -from investing_bot_framework.core.executors.execution_scheduler import ExecutionScheduler \ No newline at end of file +from investing_bot_framework.core.executors.execution_scheduler import ExecutionScheduler diff --git a/investing_bot_framework/core/executors/data_provider_executor.py b/investing_bot_framework/core/executors/data_provider_executor.py deleted file mode 100644 index 8da171de..00000000 --- a/investing_bot_framework/core/executors/data_provider_executor.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import List - -from investing_bot_framework.core.workers import Worker -from investing_bot_framework.core.data_providers import DataProvider -from investing_bot_framework.core.executors import Executor -from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS - - -class DataProviderExecutor(Executor): - """ - Class DataProviderExecutor: is an executor for DataProvider instances. - """ - - def __init__(self, data_providers: List[DataProvider] = None, max_workers: int = DEFAULT_MAX_WORKERS): - super(DataProviderExecutor, self).__init__(max_workers=max_workers) - - self._registered_data_providers: List[DataProvider] = [] - - if data_providers is not None and len(data_providers) > 0: - self._registered_data_providers = data_providers - - def create_workers(self) -> List[Worker]: - return self._registered_data_providers - - @property - def registered_data_providers(self) -> List[DataProvider]: - return self._registered_data_providers - - @property - def configured(self): - return self._registered_data_providers is not None and len(self._registered_data_providers) > 0 \ No newline at end of file From fa4a13c4bce547520c05eb2dca88b10818a8f48d Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 00:07:29 +0200 Subject: [PATCH 26/55] Add Strategy export --- investing_bot_framework/core/strategies/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 investing_bot_framework/core/strategies/__init__.py diff --git a/investing_bot_framework/core/strategies/__init__.py b/investing_bot_framework/core/strategies/__init__.py new file mode 100644 index 00000000..b971a42d --- /dev/null +++ b/investing_bot_framework/core/strategies/__init__.py @@ -0,0 +1 @@ +from investing_bot_framework.core.strategies.strategy import Strategy From 0bc6d4efc88d7fa51f840d6b577e1150eb58818e Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 00:08:05 +0200 Subject: [PATCH 27/55] Add Strategy template --- .../configuration/context.py-template | 9 +++++++-- .../data_providers/data_providers.py-template | 4 ++-- .../bot_project_template/strategies/strategies.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template index 85b4ad6f..21a454e1 100644 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template @@ -1,13 +1,18 @@ from investing_bot_framework.core.context import BotContext from investing_bot_framework.core.context.states.setup_state import SetupState -from investing_bot_framework.core.context.states.data_provider_state import DataProviderState +from investing_bot_framework.core.context.states.data_providing_state import DataProvidingState +from investing_bot_framework.core.context.states.strategy_state import StrategyState +# Import custom components from bot.data_providers.data_providers import MyDataProvider +from bot.strategies.strategies import MyStrategy # Register Initial state context = BotContext() context.register_initial_state(SetupState) # Register all data providers -DataProviderState.register_data_providers([MyDataProvider()]) +DataProvidingState.register_data_providers([MyDataProvider()]) +# Register all strategies +StrategyState.register_strategies([MyStrategy()]) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template index a9765c0b..3262f658 100755 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/data_providers/data_providers.py-template @@ -2,13 +2,13 @@ from investing_bot_framework.core.data_providers import DataProvider from investing_bot_framework.core.utils import TimeUnit """ -Define here all you data providers, e.g. Rest API client +Define here all your data providers """ class MyDataProvider(DataProvider): time_unit = TimeUnit.SECOND - time_interval = 1 + time_interval = 10 id = 'my_data_provider' def provide_data(self) -> None: diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py b/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py new file mode 100644 index 00000000..56c34fb9 --- /dev/null +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py @@ -0,0 +1,15 @@ +from investing_bot_framework.core.strategies import Strategy +from investing_bot_framework.core.utils import TimeUnit + +""" +Define here all your strategies +""" + + +class MyStrategy(Strategy): + time_unit = TimeUnit.SECOND + time_interval = 10 + id = 'my_strategy' + + def apply_strategy(self) -> None: + pass From 3b59b1506f383e41d50afac8267efd2a47483134 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 00:08:37 +0200 Subject: [PATCH 28/55] Name template --- .../strategies/{strategies.py => strategies.py-templace} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/{strategies.py => strategies.py-templace} (100%) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py b/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py-templace similarity index 100% rename from investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py rename to investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py-templace From 1a96a3fc9c2a3cdafbeda78b4333d3d884ed7e49 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 00:09:11 +0200 Subject: [PATCH 29/55] Add StrategyState --- .../core/context/states/bot_state.py | 10 +- ...vider_state.py => data_providing_state.py} | 53 ++++--- .../core/context/states/setup_state.py | 4 +- .../core/context/states/strategy_state.py | 137 ++++++++++++++++-- 4 files changed, 164 insertions(+), 40 deletions(-) rename investing_bot_framework/core/context/states/{data_provider_state.py => data_providing_state.py} (71%) diff --git a/investing_bot_framework/core/context/states/bot_state.py b/investing_bot_framework/core/context/states/bot_state.py index de777382..519b4e27 100644 --- a/investing_bot_framework/core/context/states/bot_state.py +++ b/investing_bot_framework/core/context/states/bot_state.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Type, List +from typing import List from investing_bot_framework.core.context.state_validator import StateValidator @@ -33,18 +33,10 @@ def start(self): def run(self) -> None: pass - @abstractmethod - def stop(self) -> None: - pass - @property def context(self): return self._bot_context - @abstractmethod - def reconfigure(self) -> None: - pass - def validate_state(self) -> bool: """ Function that will validate the state diff --git a/investing_bot_framework/core/context/states/data_provider_state.py b/investing_bot_framework/core/context/states/data_providing_state.py similarity index 71% rename from investing_bot_framework/core/context/states/data_provider_state.py rename to investing_bot_framework/core/context/states/data_providing_state.py index d9d829c4..201f4ba0 100644 --- a/investing_bot_framework/core/context/states/data_provider_state.py +++ b/investing_bot_framework/core/context/states/data_providing_state.py @@ -8,13 +8,39 @@ from investing_bot_framework.core.exceptions import OperationalException from investing_bot_framework.core.context.states import BotState from investing_bot_framework.core.executors import ExecutionScheduler +from investing_bot_framework.core.workers import Worker from investing_bot_framework.core.data_providers import DataProvider -from investing_bot_framework.core.executors.data_provider_executor import DataProviderExecutor +from investing_bot_framework.core.executors import Executor from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS, SETTINGS_MAX_WORKERS logger = logging.getLogger(__name__) +class DataProviderExecutor(Executor): + """ + Class DataProviderExecutor: is an executor for DataProvider instances. + """ + + def __init__(self, data_providers: List[DataProvider] = None, max_workers: int = DEFAULT_MAX_WORKERS): + super(DataProviderExecutor, self).__init__(max_workers=max_workers) + + self._registered_data_providers: List[DataProvider] = [] + + if data_providers is not None and len(data_providers) > 0: + self._registered_data_providers = data_providers + + def create_workers(self) -> List[Worker]: + return self._registered_data_providers + + @property + def registered_data_providers(self) -> List[DataProvider]: + return self._registered_data_providers + + @property + def configured(self): + return self._registered_data_providers is not None and len(self._registered_data_providers) > 0 + + class DataProviderScheduler(ExecutionScheduler): """ Data Provider scheduler that will function as a scheduler to make sure it keeps it state across multiple states, @@ -42,7 +68,7 @@ def configured(self) -> bool: return self._configured -class DataProviderState(BotState, Observer): +class DataProvidingState(BotState, Observer): """ Represent the data_providers state of a bot. This state will load all the defined data_providers providers and will run them. @@ -58,19 +84,19 @@ class DataProviderState(BotState, Observer): data_provider_scheduler: DataProviderScheduler = None def __init__(self, context: BotContext) -> None: - super(DataProviderState, self).__init__(context) + super(DataProvidingState, self).__init__(context) self._updated = False self.data_provider_executor = None def _schedule_data_providers(self) -> List[DataProvider]: - if not DataProviderState.data_provider_scheduler: - DataProviderState.data_provider_scheduler = DataProviderScheduler() + if not DataProvidingState.data_provider_scheduler: + DataProvidingState.data_provider_scheduler = DataProviderScheduler() - if not DataProviderState.data_provider_scheduler.configured: - DataProviderState.data_provider_scheduler.configure(self.registered_data_providers) + if not DataProvidingState.data_provider_scheduler.configured: + DataProvidingState.data_provider_scheduler.configure(self.registered_data_providers) - planning = DataProviderState.data_provider_scheduler.schedule_executions() + planning = DataProvidingState.data_provider_scheduler.schedule_executions() planned_data_providers = [] for data_provider in self.registered_data_providers: @@ -113,22 +139,13 @@ def run(self) -> None: for data_provider in self.data_provider_executor.registered_data_providers: logger.info("Data provider: {} finished running".format(data_provider.get_id())) - def stop(self) -> None: - """ - Stop all data_providers providers - """ - pass - @synchronized def update(self, observable, **kwargs) -> None: self._updated = True @staticmethod def register_data_providers(data_providers: List) -> None: - DataProviderState.registered_data_providers = data_providers - - def reconfigure(self) -> None: - pass + DataProvidingState.registered_data_providers = data_providers diff --git a/investing_bot_framework/core/context/states/setup_state.py b/investing_bot_framework/core/context/states/setup_state.py index 6dc1061f..4bc1ff7a 100644 --- a/investing_bot_framework/core/context/states/setup_state.py +++ b/investing_bot_framework/core/context/states/setup_state.py @@ -4,8 +4,8 @@ class SetupState(BotState): - from investing_bot_framework.core.context.states.data_provider_state import DataProviderState - transition_state_class = DataProviderState + from investing_bot_framework.core.context.states.data_providing_state import DataProvidingState + transition_state_class = DataProvidingState def __init__(self, context): super(SetupState, self).__init__(context) diff --git a/investing_bot_framework/core/context/states/strategy_state.py b/investing_bot_framework/core/context/states/strategy_state.py index 94b6c719..0d25874f 100644 --- a/investing_bot_framework/core/context/states/strategy_state.py +++ b/investing_bot_framework/core/context/states/strategy_state.py @@ -1,21 +1,136 @@ -from typing import Type +import logging +import time +from typing import List + from investing_bot_framework.core.context.states import BotState +from investing_bot_framework.core.executors import ExecutionScheduler +from investing_bot_framework.core.exceptions import OperationalException +from investing_bot_framework.core.workers import Worker +from investing_bot_framework.core.executors import Executor +from investing_bot_framework.core.events import Observer +from investing_bot_framework.core.configuration.config_constants import DEFAULT_MAX_WORKERS, SETTINGS_MAX_WORKERS + +logger = logging.getLogger(__name__) + + +class StrategyExecutor(Executor): + """ + Class StrategyExecutor: is an executor for Strategy instances. + """ + + def __init__(self, strategies: List = None, max_workers: int = DEFAULT_MAX_WORKERS): + super(StrategyExecutor, self).__init__(max_workers=max_workers) + + self._registered_strategies: List = [] + + if strategies is not None and len(strategies) > 0: + self._registered_strategies = strategies + + def create_workers(self) -> List[Worker]: + return self._registered_strategies + + @property + def registered_strategies(self) -> List: + return self._registered_strategies + + @property + def configured(self): + return self._registered_strategies is not None and len(self._registered_strategies) > 0 + + +class StrategyScheduler(ExecutionScheduler): + """ + Strategy scheduler that will function as a scheduler. + """ + + def __init__(self): + self._configured = False + super(StrategyScheduler, self).__init__() + + def configure(self, strategies: List) -> None: + self._planning = {} + + for strategy in strategies: + self.add_execution_task( + execution_id=strategy.get_id(), + time_unit=strategy.get_time_unit(), + interval=strategy.get_time_interval() + ) + + self._configured = True + + @property + def configured(self) -> bool: + return self._configured -class StrategyState(BotState): +class StrategyState(BotState, Observer): + + from investing_bot_framework.core.context.states.data_providing_state import DataProvidingState + transition_state_class = DataProvidingState + + registered_strategies: List = None + strategy_scheduler: StrategyScheduler = None + + def __init__(self, context): + super(StrategyState, self).__init__(context) + self._updated = False + self.strategy_executor = None + + def _schedule_strategies(self) -> List: + + if not StrategyState.strategy_scheduler: + StrategyState.strategy_scheduler = StrategyScheduler() + + if not StrategyState.strategy_scheduler.configured: + StrategyState.strategy_scheduler.configure(self.registered_strategies) + + planning = StrategyState.strategy_scheduler.schedule_executions() + planned_strategies = [] + + for strategy in self.registered_strategies: + + if strategy.get_id() in planning: + planned_strategies.append(strategy) + + return planned_strategies + + def _start_strategies(self, strategies: List) -> None: + + self.strategy_executor = StrategyExecutor( + strategies=strategies, + max_workers=self.context.settings.get(SETTINGS_MAX_WORKERS, DEFAULT_MAX_WORKERS) + ) + + if self.strategy_executor.configured: + self.strategy_executor.add_observer(self) + self.strategy_executor.start() + else: + # Skip the execution + self._updated = True def run(self) -> None: - pass - def stop(self) -> None: - pass + if self.registered_strategies is None: + raise OperationalException("Data providing state has not any data providers configured") + + # Schedule the strategies providers + planned_strategies = self._schedule_strategies() + + # Execute all the strategies providers + self._start_strategies(planned_strategies) + + # Sleep till updated + while not self._updated: + time.sleep(1) - def reconfigure(self) -> None: - pass + # Collect all strategies from the strategies providers + for strategies in self.strategy_executor.registered_strategies: + logger.info("Data provider: {} finished running".format(strategies.get_id())) def update(self, observable, **kwargs) -> None: - pass + self._updated = True - def get_transition_state_class(self) -> Type: - from investing_bot_framework.core.context.states.data_provider_state import DataProviderState - return DataProviderState + @staticmethod + def register_strategies(strategies: List) -> None: + StrategyState.registered_strategies = strategies From 7f06942757de57b809729c0baaa94c78d02f580b Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:43:55 +0200 Subject: [PATCH 30/55] Format spacing at the end of the file --- investing_bot_framework/core/events/observable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/investing_bot_framework/core/events/observable.py b/investing_bot_framework/core/events/observable.py index 802dd16a..27c766df 100644 --- a/investing_bot_framework/core/events/observable.py +++ b/investing_bot_framework/core/events/observable.py @@ -30,4 +30,4 @@ def notify_observers(self, **kwargs) -> None: @property def observers(self) -> List[Observer]: - return self._observers \ No newline at end of file + return self._observers From aceba96b355631104e7cd30edfdb571512cd5fdf Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:45:04 +0200 Subject: [PATCH 31/55] Remove unused methods --- .../core/context/bot_context.py | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/investing_bot_framework/core/context/bot_context.py b/investing_bot_framework/core/context/bot_context.py index 96eb5c1c..38cc6ba6 100644 --- a/investing_bot_framework/core/context/bot_context.py +++ b/investing_bot_framework/core/context/bot_context.py @@ -3,7 +3,7 @@ from investing_bot_framework.core.configuration import settings from investing_bot_framework.core.exceptions import OperationalException from investing_bot_framework.core.utils import Singleton -from investing_bot_framework.core.context.states import BotState +from investing_bot_framework.core.states import BotState class BotContext(metaclass=Singleton): @@ -19,11 +19,6 @@ class BotContext(metaclass=Singleton): settings = settings def register_initial_state(self, bot_state: Type[BotState]) -> None: - - # Stop the current state of the investing_bot_framework - if self._state: - self._state.stop() - self._state = bot_state(context=self) def transition_to(self, bot_state: Type[BotState]) -> None: @@ -65,18 +60,3 @@ def _run_state(self) -> None: transition_state = self._state.get_transition_state_class() self.transition_to(transition_state) - def stop(self) -> None: - """ - Stop the current state of the investing_bot_framework - """ - - self._check_state(raise_exception=True) - self._state.stop() - - def reconfigure(self) -> None: - """ - Reconfigure the current state of the investing_bot_framework - """ - - self._check_state(raise_exception=True) - self._state.reconfigure() From 135a445b4db6c46b5701aa8b6544159ca3e16ce9 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:45:33 +0200 Subject: [PATCH 32/55] Initial commit for order executors --- investing_bot_framework/core/order_executors/__init__.py | 1 + .../core/order_executors/order_executor.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 investing_bot_framework/core/order_executors/__init__.py create mode 100644 investing_bot_framework/core/order_executors/order_executor.py diff --git a/investing_bot_framework/core/order_executors/__init__.py b/investing_bot_framework/core/order_executors/__init__.py new file mode 100644 index 00000000..1af709d0 --- /dev/null +++ b/investing_bot_framework/core/order_executors/__init__.py @@ -0,0 +1 @@ +from investing_bot_framework.core.order_executors.order_executor import OrderExecutor diff --git a/investing_bot_framework/core/order_executors/order_executor.py b/investing_bot_framework/core/order_executors/order_executor.py new file mode 100644 index 00000000..552f20d8 --- /dev/null +++ b/investing_bot_framework/core/order_executors/order_executor.py @@ -0,0 +1,8 @@ +from abc import ABC, abstractmethod + + +class OrderExecutor(ABC): + + @abstractmethod + def execute_orders(self) -> None: + pass From 6d0a7fc6cdcfeb3d9c7f1900c83ab84e6dc57ab7 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:46:55 +0200 Subject: [PATCH 33/55] Put states in dedicated folder --- .../core/states/templates/__init__.py | 0 .../templates}/data_providing_state.py | 10 +++++----- .../states => states/templates}/setup_state.py | 16 +++++++--------- .../templates}/strategy_state.py | 12 +++++++----- 4 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 investing_bot_framework/core/states/templates/__init__.py rename investing_bot_framework/core/{context/states => states/templates}/data_providing_state.py (95%) rename investing_bot_framework/core/{context/states => states/templates}/setup_state.py (69%) rename investing_bot_framework/core/{context/states => states/templates}/strategy_state.py (93%) diff --git a/investing_bot_framework/core/states/templates/__init__.py b/investing_bot_framework/core/states/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/investing_bot_framework/core/context/states/data_providing_state.py b/investing_bot_framework/core/states/templates/data_providing_state.py similarity index 95% rename from investing_bot_framework/core/context/states/data_providing_state.py rename to investing_bot_framework/core/states/templates/data_providing_state.py index 201f4ba0..074caa1a 100644 --- a/investing_bot_framework/core/context/states/data_providing_state.py +++ b/investing_bot_framework/core/states/templates/data_providing_state.py @@ -6,7 +6,7 @@ from investing_bot_framework.core.events import Observer from investing_bot_framework.core.context.bot_context import BotContext from investing_bot_framework.core.exceptions import OperationalException -from investing_bot_framework.core.context.states import BotState +from investing_bot_framework.core.states import BotState from investing_bot_framework.core.executors import ExecutionScheduler from investing_bot_framework.core.workers import Worker from investing_bot_framework.core.data_providers import DataProvider @@ -78,7 +78,7 @@ class DataProvidingState(BotState, Observer): registered_data_providers: List = None - from investing_bot_framework.core.context.states.strategy_state import StrategyState + from investing_bot_framework.core.states.templates.strategy_state import StrategyState transition_state_class = StrategyState data_provider_scheduler: DataProviderScheduler = None @@ -88,6 +88,9 @@ def __init__(self, context: BotContext) -> None: self._updated = False self.data_provider_executor = None + if self.registered_data_providers is None or len(self.registered_data_providers) < 1: + raise OperationalException("Data providing state has not any data providers configured") + def _schedule_data_providers(self) -> List[DataProvider]: if not DataProvidingState.data_provider_scheduler: @@ -122,9 +125,6 @@ def _start_data_providers(self, data_providers: List[DataProvider]) -> None: def run(self) -> None: - if self.registered_data_providers is None: - raise OperationalException("Data providing state has not any data providers configured") - # Schedule the data_providers providers planned_data_providers = self._schedule_data_providers() diff --git a/investing_bot_framework/core/context/states/setup_state.py b/investing_bot_framework/core/states/templates/setup_state.py similarity index 69% rename from investing_bot_framework/core/context/states/setup_state.py rename to investing_bot_framework/core/states/templates/setup_state.py index 4bc1ff7a..96ff6df8 100644 --- a/investing_bot_framework/core/context/states/setup_state.py +++ b/investing_bot_framework/core/states/templates/setup_state.py @@ -1,15 +1,19 @@ +import logging + from investing_bot_framework.core.exceptions import ImproperlyConfigured -from investing_bot_framework.core.context.states import BotState +from investing_bot_framework.core.states import BotState + +logger = logging.getLogger(__name__) class SetupState(BotState): - from investing_bot_framework.core.context.states.data_providing_state import DataProvidingState + from investing_bot_framework.core.states.templates.data_providing_state import DataProvidingState transition_state_class = DataProvidingState def __init__(self, context): super(SetupState, self).__init__(context) - + def run(self) -> None: """ Running the setup state. @@ -27,10 +31,4 @@ def run(self) -> None: "manage.py file" ) - def stop(self) -> None: - # Stopping all services - pass - def reconfigure(self) -> None: - # Clean up and reconfigure all the services - pass diff --git a/investing_bot_framework/core/context/states/strategy_state.py b/investing_bot_framework/core/states/templates/strategy_state.py similarity index 93% rename from investing_bot_framework/core/context/states/strategy_state.py rename to investing_bot_framework/core/states/templates/strategy_state.py index 0d25874f..fb01c241 100644 --- a/investing_bot_framework/core/context/states/strategy_state.py +++ b/investing_bot_framework/core/states/templates/strategy_state.py @@ -2,7 +2,7 @@ import time from typing import List -from investing_bot_framework.core.context.states import BotState +from investing_bot_framework.core.states import BotState from investing_bot_framework.core.executors import ExecutionScheduler from investing_bot_framework.core.exceptions import OperationalException from investing_bot_framework.core.workers import Worker @@ -66,9 +66,6 @@ def configured(self) -> bool: class StrategyState(BotState, Observer): - from investing_bot_framework.core.context.states.data_providing_state import DataProvidingState - transition_state_class = DataProvidingState - registered_strategies: List = None strategy_scheduler: StrategyScheduler = None @@ -126,7 +123,7 @@ def run(self) -> None: # Collect all strategies from the strategies providers for strategies in self.strategy_executor.registered_strategies: - logger.info("Data provider: {} finished running".format(strategies.get_id())) + logger.info("Strategy: {} finished running".format(strategies.get_id())) def update(self, observable, **kwargs) -> None: self._updated = True @@ -134,3 +131,8 @@ def update(self, observable, **kwargs) -> None: @staticmethod def register_strategies(strategies: List) -> None: StrategyState.registered_strategies = strategies + + def get_transition_state_class(self): + from investing_bot_framework.core.states.templates.ordering_state import OrderingState + return OrderingState + From 6dc4e762a41d2d747c7a11546b5263b8b8aa5949 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:47:22 +0200 Subject: [PATCH 34/55] Initial commit for OrderingState --- .../core/states/templates/ordering_state.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 investing_bot_framework/core/states/templates/ordering_state.py diff --git a/investing_bot_framework/core/states/templates/ordering_state.py b/investing_bot_framework/core/states/templates/ordering_state.py new file mode 100644 index 00000000..c1e5763d --- /dev/null +++ b/investing_bot_framework/core/states/templates/ordering_state.py @@ -0,0 +1,30 @@ +from typing import List + +from investing_bot_framework.core.states import BotState +from investing_bot_framework.core.exceptions import OperationalException +from investing_bot_framework.core.order_executors import OrderExecutor + + +class OrderingState(BotState): + + from investing_bot_framework.core.states.templates.data_providing_state import DataProvidingState + transition_state_class = DataProvidingState + + order_executors: List[OrderExecutor] = None + + def __init__(self, context) -> None: + super(OrderingState, self).__init__(context) + + if self.order_executors is None or len(self.order_executors) < 1: + raise OperationalException("OrderingState state has not any order executors configured") + + def run(self) -> None: + + for order_executor in OrderingState.order_executors: + order_executor.execute_orders() + + @staticmethod + def register_order_executors(order_executors: List[OrderExecutor]) -> None: + OrderingState.order_executors = order_executors + + From 4874e1b323b7908a5a35c314ee89a5bb6292bad3 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:47:54 +0200 Subject: [PATCH 35/55] Put BotState in dedicated folder --- .../core/states/__init__.py | 2 ++ .../core/{context => }/states/bot_state.py | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 investing_bot_framework/core/states/__init__.py rename investing_bot_framework/core/{context => }/states/bot_state.py (63%) diff --git a/investing_bot_framework/core/states/__init__.py b/investing_bot_framework/core/states/__init__.py new file mode 100644 index 00000000..43b869e2 --- /dev/null +++ b/investing_bot_framework/core/states/__init__.py @@ -0,0 +1,2 @@ +from investing_bot_framework.core.states.bot_state import BotState + diff --git a/investing_bot_framework/core/context/states/bot_state.py b/investing_bot_framework/core/states/bot_state.py similarity index 63% rename from investing_bot_framework/core/context/states/bot_state.py rename to investing_bot_framework/core/states/bot_state.py index 519b4e27..884d4d89 100644 --- a/investing_bot_framework/core/context/states/bot_state.py +++ b/investing_bot_framework/core/states/bot_state.py @@ -14,14 +14,18 @@ class BotState(ABC): transition_state_class = None # Validator for the current state - state_validators = None + pre_state_validators: List[StateValidator] = None + post_state_validators: List[StateValidator] = None - def __init__(self, context, state_validator: StateValidator = None) -> None: + def __init__(self, context) -> None: self._bot_context = context - self._state_validator = state_validator def start(self): + # Will stop the state if pre-conditions are not met + if not self.validate_state(): + return + while True: self.run() @@ -37,12 +41,15 @@ def run(self) -> None: def context(self): return self._bot_context - def validate_state(self) -> bool: + def validate_state(self, pre_state: bool = False) -> bool: """ Function that will validate the state """ - state_validators = self.get_state_validators() + if pre_state: + state_validators = self.get_pre_state_validators() + else: + state_validators = self.get_post_state_validators() if state_validators is None: return True @@ -63,12 +70,18 @@ def get_transition_state_class(self): return self.transition_state_class - def get_state_validators(self) -> List[StateValidator]: + def get_pre_state_validators(self) -> List[StateValidator]: - if self.state_validators is not None: + if self.pre_state_validators is not None: return [ - state_validator() for state_validator in getattr(self, 'state_validators') + state_validator() for state_validator in getattr(self, 'pre_state_validators') if issubclass(state_validator, StateValidator) ] + def get_post_state_validators(self) -> List[StateValidator]: + if self.post_state_validators is not None: + return [ + state_validator() for state_validator in getattr(self, 'post_state_validators') + if issubclass(state_validator, StateValidator) + ] From 59f4076a0c21098a23a0bdef6718b8838b45550b Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:48:39 +0200 Subject: [PATCH 36/55] Add order_executors to templates --- .../configuration/context.py-template | 11 ++++++++--- .../configuration/settings.py-template | 7 ++++++- .../order_executors/__init__.py-template | 0 .../order_executors/order_executors.py-template | 7 +++++++ ...{strategies.py-templace => strategies.py-template} | 0 5 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/__init__.py-template create mode 100644 investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/order_executors.py-template rename investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/{strategies.py-templace => strategies.py-template} (100%) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template index 21a454e1..270d3346 100644 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/context.py-template @@ -1,11 +1,13 @@ from investing_bot_framework.core.context import BotContext -from investing_bot_framework.core.context.states.setup_state import SetupState -from investing_bot_framework.core.context.states.data_providing_state import DataProvidingState -from investing_bot_framework.core.context.states.strategy_state import StrategyState +from investing_bot_framework.core.states.templates.setup_state import SetupState +from investing_bot_framework.core.states.templates.data_providing_state import DataProvidingState +from investing_bot_framework.core.states.templates.strategy_state import StrategyState +from investing_bot_framework.core.states.templates.ordering_state import OrderingState # Import custom components from bot.data_providers.data_providers import MyDataProvider from bot.strategies.strategies import MyStrategy +from bot.order_executors.order_executors import MyOrderExecutor # Register Initial state context = BotContext() @@ -16,3 +18,6 @@ DataProvidingState.register_data_providers([MyDataProvider()]) # Register all strategies StrategyState.register_strategies([MyStrategy()]) + +# Register all order executors +OrderingState.register_order_executors([MyOrderExecutor()]) diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template index 7df34664..3aeaef9f 100755 --- a/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/configuration/settings.py-template @@ -12,7 +12,12 @@ BASE_DIR = str(Path(__file__).parent.parent) LOG_FILE_NAME = 'log' -LOG_PATH = "{}{}.log".format(BASE_DIR, LOG_FILE_NAME) +LOG_DIR = '{}/logs'.format(BASE_DIR) + +LOG_PATH = "{}/{}.log".format(LOG_DIR, LOG_FILE_NAME) + +if not os.path.isdir(LOG_DIR): + os.mkdir(LOG_DIR) if DEBUG: logging_level = "DEBUG" diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/__init__.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/__init__.py-template new file mode 100644 index 00000000..e69de29b diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/order_executors.py-template b/investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/order_executors.py-template new file mode 100644 index 00000000..e6e8d42a --- /dev/null +++ b/investing_bot_framework/templates/bot_project_directory/bot_project_template/order_executors/order_executors.py-template @@ -0,0 +1,7 @@ +from investing_bot_framework.core.order_executors import OrderExecutor + + +class MyOrderExecutor(OrderExecutor): + + def execute_orders(self) -> None: + pass diff --git a/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py-templace b/investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py-template similarity index 100% rename from investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py-templace rename to investing_bot_framework/templates/bot_project_directory/bot_project_template/strategies/strategies.py-template From e6419e28b419304a98cabece2a1f49531c3a1084 Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 16:49:54 +0200 Subject: [PATCH 37/55] Remove states from context --- Remove_states_from_context.patch | 6 ++++++ investing_bot_framework/core/context/states/__init__.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 Remove_states_from_context.patch delete mode 100644 investing_bot_framework/core/context/states/__init__.py diff --git a/Remove_states_from_context.patch b/Remove_states_from_context.patch new file mode 100644 index 00000000..ee5a184b --- /dev/null +++ b/Remove_states_from_context.patch @@ -0,0 +1,6 @@ +Index: investing_bot_framework/core/context/states/__init__.py +=================================================================== +--- investing_bot_framework/core/context/states/__init__.py (revision 59f4076a0c21098a23a0bdef6718b8838b45550b) ++++ investing_bot_framework/core/context/states/__init__.py (revision 59f4076a0c21098a23a0bdef6718b8838b45550b) +@@ -1,1 +0,0 @@ +-from investing_bot_framework.core.context.states.bot_state import BotState diff --git a/investing_bot_framework/core/context/states/__init__.py b/investing_bot_framework/core/context/states/__init__.py deleted file mode 100644 index 9511ad24..00000000 --- a/investing_bot_framework/core/context/states/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from investing_bot_framework.core.context.states.bot_state import BotState From a72c903f8300b618b79aa66cd518e1883a4fe72e Mon Sep 17 00:00:00 2001 From: sadays Date: Sun, 14 Jun 2020 17:30:08 +0200 Subject: [PATCH 38/55] Update readme --- README.md | 76 +++++++++++++------------------------------------------ 1 file changed, 18 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index d9b58d0a..907e57d2 100644 --- a/README.md +++ b/README.md @@ -1,69 +1,30 @@ -# Investing bot +# Investing Algorithm Framework -The investing bot is a free and open source investing bot written in Python. The goal is to give you a configurable bot -where you can decide on how you implement your data providers, strategies, and brokers/exchanges. Also we want to allow -you to let your bot facilitate multiple users. +The Investing Algorithm Framework is a free and open source Python framework that encourages rapid development and clean, +pragmatic design. -It is designed to be controlled via Telegram. As of now, we are aiming to make the configuration of the different -components by the use of plugins. Please see the documentation on how to make your own plugin. +The goal is to give you a configurable investing algorithm where you can decide how you implement your data providers, +strategies, and order executors. -### Disclaimer -This software is for educational purposes only. Do not risk money which you are afraid to lose. We can't stress this -enough: BEFORE YOU START USING MONEY WITH THE BOT, MAKE SURE THAT YOU TESTED YOU STRATEGIES AND DATA PROVIDERS. -USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. +#####Disclaimer +If you use this framework for your investments, do not risk money which you are afraid to lose. We can't stress this +enough: -Always start by running a investing bot in Dry-run and do not engage money before you understand how it works and what profit/loss you should expect. - -We strongly recommend you to have coding and Python knowledge, or trust the people that created the plugins your using. -Do not hesitate to read the source code and understand the mechanism of this bot or the plugin you're using. - -Brokers/Exchange marketplaces supported ------- -Will be updated in the future +BEFORE YOU START USING MONEY WITH THE FRAMEWORK, MAKE SURE THAT YOU TESTED YOUR COMPONENTS THOROUGHLY. USE THE SOFTWARE AT +YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR INVESTMENT RESULTS. +Also, make sure that you read the source code of any plugin you use or implementation of an algorithm made with this +framework. Documentation ------ -Will be updated in the future - -## Features - -- [x] **Based on Python 3.6+**: Support for all operating systems - Windows, macOS and Linux. -- [x] **Persistence**: Persistence is achieved through sqlite. -- [ ] **Dry-run**: Run the bot without playing money. -- [ ] **REST API**: Manage the bot with the use of a REST API. -- [ ] **Backtesting**: Run a simulation of your buy/sell strategy. -- [ ] **Manageable via Telegram**: Manage the bot with Telegram. -- [ ] **Display profit/loss**: Display your profit/loss. -- [ ] **Daily summary of profit/loss**: Provide a daily summary of your profit/loss. -- [ ] **Performance status report**: Provide a performance status of your current trades. - -## Quick start - -The investing bot provides a Linux/macOS script to install all dependencies and help you to configure the bot. - -The script will come as a future update - -### Bot commands - - -``` -usage: main.py [-h] [-V] [-c PATH] - -Trading bot based on value principles - -optional arguments: - -h, --help show this help message and exit - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). - -``` - -### Telegram RPC commands - -Telegram is not mandatory. However, this is a great way to control your bot. +All documentation is in the "docs" directory and online at "". If you're just getting started, here's how we recommend +you read the docs: +* First, read install for instructions on installing Investing Algorithm Framework. +* Next, work through the tutorials in order. ("Quickstart", "Template algorithm", "Custom algorithm"). +* For concrete algorithm examples you probably want to read through the topical guides. + ## Development branches @@ -74,7 +35,6 @@ The project is currently setup in two main branches: - `feature/*` - These are feature branches, which are being worked on heavily. Please don't use these unless you want to test a specific feature. - `hotfix/*` - These are hot fix branches, which are being worked on heavily. Please don't use these unless you really need to. -## Support ### Help / Slack From a9bf50a5d57e1176d575c890d1b419548a70be9b Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 16:24:31 +0200 Subject: [PATCH 39/55] Add version utils --- investing_bot_framework/__init__.py | 4 ++++ investing_bot_framework/utils/__init__.py | 0 investing_bot_framework/utils/version.py | 25 +++++++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 investing_bot_framework/utils/__init__.py create mode 100644 investing_bot_framework/utils/version.py diff --git a/investing_bot_framework/__init__.py b/investing_bot_framework/__init__.py index e69de29b..df5df79d 100644 --- a/investing_bot_framework/__init__.py +++ b/investing_bot_framework/__init__.py @@ -0,0 +1,4 @@ +from investing_bot_framework.utils.version import get_version + +VERSION = (1, 0, 0, 'alpha', 0) + diff --git a/investing_bot_framework/utils/__init__.py b/investing_bot_framework/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/investing_bot_framework/utils/version.py b/investing_bot_framework/utils/version.py new file mode 100644 index 00000000..c908e667 --- /dev/null +++ b/investing_bot_framework/utils/version.py @@ -0,0 +1,25 @@ +def get_version(version=None): + version = get_complete_version(version) + main = get_main_version(version) + return main + + +def get_main_version(version=None): + """Return main version (X.Y[.Z]) from VERSION.""" + version = get_complete_version(version) + parts = 2 if version[2] == 0 else 3 + return '.'.join(str(x) for x in version[:parts]) + + +def get_complete_version(version=None): + """ + Return a tuple of the investing algorithm framework version. If version argument is non-empty, + check for correctness of the tuple provided. + """ + if version is None: + from investing_bot_framework import VERSION as version + else: + assert len(version) == 5 + assert version[3] in ('alpha', 'beta', 'rc', 'final') + + return version From 6cefc21438c437be6e39949ed4e5b4c997fb3584 Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 19:34:45 +0200 Subject: [PATCH 40/55] Add version tests --- ci/test.sh | 3 ++ .../tests/resources/standard_settings.py | 47 +++++++++++++++++-- .../tests/utils/__init__.py | 0 .../tests/utils/test_version.py | 14 ++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 ci/test.sh create mode 100644 investing_bot_framework/tests/utils/__init__.py create mode 100644 investing_bot_framework/tests/utils/test_version.py diff --git a/ci/test.sh b/ci/test.sh new file mode 100644 index 00000000..3b72dcf3 --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +python3 -m unittest discover -s ../ diff --git a/investing_bot_framework/tests/resources/standard_settings.py b/investing_bot_framework/tests/resources/standard_settings.py index 97e7135a..71c220d8 100644 --- a/investing_bot_framework/tests/resources/standard_settings.py +++ b/investing_bot_framework/tests/resources/standard_settings.py @@ -1,10 +1,49 @@ +import os from pathlib import Path -BOT_PROJECT_NAME = 'TEST_ALGORITHM' +BOT_PROJECT_NAME = 'bot' -BASE_DIR = str(Path(__file__).parent) +BOT_CONTEXT_CONFIGURATION = 'bot.configuration.context' -INSTALLED_DATA_PROVIDER_APPS = [] +# Change this when not in development, feature or hot-fix branch +DEBUG = int(os.environ.get('DEBUG', True)) -INSTALLED_STRATEGY_APPS = [] +BASE_DIR = str(Path(__file__).parent.parent) +LOG_FILE_NAME = 'log' + +LOG_DIR = '{}/logs'.format(BASE_DIR) + +LOG_PATH = "{}/{}.log".format(LOG_DIR, LOG_FILE_NAME) + +# if not os.path.isdir(LOG_DIR): +# os.mkdir(LOG_DIR) + +if DEBUG: + logging_level = "DEBUG" +else: + logging_level = "INFO" + +# Logging configuration +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(levelname)s %(asctime)s - [thread: %(threadName)-4s %(name)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'standard', + }, + }, + 'loggers': { + '': { + 'level': logging_level, + 'handlers': ['console'], + }, + }, +} \ No newline at end of file diff --git a/investing_bot_framework/tests/utils/__init__.py b/investing_bot_framework/tests/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/investing_bot_framework/tests/utils/test_version.py b/investing_bot_framework/tests/utils/test_version.py new file mode 100644 index 00000000..8195abfd --- /dev/null +++ b/investing_bot_framework/tests/utils/test_version.py @@ -0,0 +1,14 @@ +from unittest import TestCase +from investing_bot_framework.utils.version import get_version, get_complete_version, get_main_version + + +class Version(TestCase): + + def test(self): + self.assertIsNotNone(get_version()) + self.assertEqual(type(get_version()), str) + + version = (1, 0, 0, 'alpha', 0) + self.assertEqual(get_version(version), '1.0') + self.assertEqual(get_main_version(version), '1.0') + self.assertEqual(get_complete_version(version), version) From 7525ba397ef391678b00690b9887c48023bb0480 Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 19:41:52 +0200 Subject: [PATCH 41/55] Initial travis.ci definition --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..3d9a8749 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python + +install: + - pip install -r requirements.txt + +script: + - ./ci/test.sh \ No newline at end of file From ccec2f4d815170f9b9b6e6478ac3b24b874439fc Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 19:44:43 +0200 Subject: [PATCH 42/55] Add execution of test.sh --- ci/test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/test.sh diff --git a/ci/test.sh b/ci/test.sh old mode 100644 new mode 100755 From 820974d0108f56b9ac0be3cb11c87d170482618c Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 19:48:55 +0200 Subject: [PATCH 43/55] Update travis.ci definition --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3d9a8749..841b5b0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: python +before_install: + - echo | ls . install: - pip install -r requirements.txt From 169368fd3b58ad506904a3e4c537a0b0e9cbf701 Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 19:51:29 +0200 Subject: [PATCH 44/55] Update travis.ci definition --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 841b5b0e..a6024b9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python before_install: - echo | ls . - + - echo | ls ./investing_bot_framework/ install: - pip install -r requirements.txt From dc71efb1659969bd0fb225229456a4800936f837 Mon Sep 17 00:00:00 2001 From: DUYN Date: Thu, 18 Jun 2020 19:56:52 +0200 Subject: [PATCH 45/55] Update travis.ci definition --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a6024b9b..c1855ca2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python before_install: - echo | ls . - - echo | ls ./investing_bot_framework/ + - echo | ls ./investing_bot_framework/tests/core/executors install: - pip install -r requirements.txt From da5fd09740d24586a7654f98359c634a4fc710a7 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 10:05:16 +0200 Subject: [PATCH 46/55] Update travis.ci definition --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1855ca2..3dddcde5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ language: python + before_install: - - echo | ls . - - echo | ls ./investing_bot_framework/tests/core/executors + - virtualenv venv + - source venv/bin/activate + install: - pip install -r requirements.txt From b7222070692917026bc635c871f4f0cb77bb1a0b Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 11:45:58 +0200 Subject: [PATCH 47/55] Update travis.ci definition --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3dddcde5..f868eb03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,4 @@ before_install: install: - pip install -r requirements.txt -script: - - ./ci/test.sh \ No newline at end of file +script: pytest # run test From 7d35c3e2ac48818db86ef14acde498a9a626268a Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 11:47:38 +0200 Subject: [PATCH 48/55] Update travis.ci definition --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f868eb03..b8f22ae5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: python -before_install: - - virtualenv venv - - source venv/bin/activate - install: - pip install -r requirements.txt From 2462e4059deab8a20f2dff3fca8650282f171ac8 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 11:48:42 +0200 Subject: [PATCH 49/55] Change to pytest framework --- .../tests/utils/test_version.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/investing_bot_framework/tests/utils/test_version.py b/investing_bot_framework/tests/utils/test_version.py index 8195abfd..9902dc07 100644 --- a/investing_bot_framework/tests/utils/test_version.py +++ b/investing_bot_framework/tests/utils/test_version.py @@ -1,14 +1,13 @@ -from unittest import TestCase from investing_bot_framework.utils.version import get_version, get_complete_version, get_main_version -class Version(TestCase): +def test(): + assert get_version() is not None + assert type(get_version()) == str + + version = (1, 0, 0, 'alpha', 0) + assert get_version(version) == '1.0' + assert get_main_version(version) == '1.0' + assert get_complete_version(version) == version - def test(self): - self.assertIsNotNone(get_version()) - self.assertEqual(type(get_version()), str) - version = (1, 0, 0, 'alpha', 0) - self.assertEqual(get_version(version), '1.0') - self.assertEqual(get_main_version(version), '1.0') - self.assertEqual(get_complete_version(version), version) From 7bf0d66b50f9005327bb9a842b65460e18858757 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 11:48:54 +0200 Subject: [PATCH 50/55] Change to pytest framework --- .../core/data/data_providers/data_provider.py | 74 +--- .../core/data/data_providers/resources.py | 29 ++ .../tests/core/executors/resources.py | 40 ++ .../tests/core/executors/test_executor.py | 376 +++++++++--------- .../tests/core/executors/test_scheduler.py | 60 +-- .../core/resolvers/test_database_resolver.py | 27 +- 6 files changed, 322 insertions(+), 284 deletions(-) create mode 100644 investing_bot_framework/tests/core/data/data_providers/resources.py create mode 100644 investing_bot_framework/tests/core/executors/resources.py diff --git a/investing_bot_framework/tests/core/data/data_providers/data_provider.py b/investing_bot_framework/tests/core/data/data_providers/data_provider.py index 28dac426..d94326cd 100644 --- a/investing_bot_framework/tests/core/data/data_providers/data_provider.py +++ b/investing_bot_framework/tests/core/data/data_providers/data_provider.py @@ -1,64 +1,34 @@ -from typing import Dict, Any -from unittest import TestCase +from investing_bot_framework.tests.core.data.data_providers.resources import TestDataProviderOne, \ + TestDataProviderTwo, TestObserver -from investing_bot_framework.core.data_providers import DataProvider -from investing_bot_framework.core.events import Observer +def test(): + data_provider_one = TestDataProviderOne() -class TestDataProviderOne(DataProvider): + assert data_provider_one.id is not None + assert data_provider_one.get_id() == TestDataProviderOne.id - id = 'TestDataProviderOne' + observer = TestObserver() + data_provider_one.add_observer(observer) - def provide_data(self, **kwargs: Dict[str, Any]) -> Any: - return "data_providers" + # Run the data_providers provider + data_provider_one.start() + # Observer must have been updated + assert observer.update_count == 1 -class TestDataProviderTwo(DataProvider): + data_provider_two = TestDataProviderTwo() - id = 'TestDataProviderTwo' + assert data_provider_two.id is not None + assert data_provider_two.get_id() == TestDataProviderTwo.id - def provide_data(self, **kwargs: Dict[str, Any]) -> Any: - return "data_providers" + data_provider_two.add_observer(observer) + # Run the data_providers provider + data_provider_two.start() -class TestObserver(Observer): + # Observer must have been updated + assert observer.update_count == 2 - def __init__(self) -> None: - self.update_count = 0 - - def update(self, observable, **kwargs) -> None: - self.update_count += 1 - - -class DataProviderSetup(TestCase): - - def test(self): - data_provider_one = TestDataProviderOne() - - self.assertIsNotNone(data_provider_one.id) - self.assertEqual(data_provider_one.get_id(), TestDataProviderOne.id) - - observer = TestObserver() - data_provider_one.add_observer(observer) - - # Run the data_providers provider - data_provider_one.start() - - # Observer must have been updated - self.assertEqual(observer.update_count, 1) - - data_provider_two = TestDataProviderTwo() - - self.assertIsNotNone(data_provider_two.id) - self.assertEqual(data_provider_two.get_id(), TestDataProviderTwo.id) - - data_provider_two.add_observer(observer) - - # Run the data_providers provider - data_provider_two.start() - - # Observer must have been updated - self.assertEqual(observer.update_count, 2) - - # Id´s must be different - self.assertNotEqual(TestDataProviderOne.id, TestDataProviderTwo.id) \ No newline at end of file + # Id´s must be different + assert TestDataProviderOne.id != TestDataProviderTwo.id diff --git a/investing_bot_framework/tests/core/data/data_providers/resources.py b/investing_bot_framework/tests/core/data/data_providers/resources.py new file mode 100644 index 00000000..6a8754a2 --- /dev/null +++ b/investing_bot_framework/tests/core/data/data_providers/resources.py @@ -0,0 +1,29 @@ +from typing import Dict, Any + +from investing_bot_framework.core.events import Observer +from investing_bot_framework.core.data_providers import DataProvider + + +class TestDataProviderOne(DataProvider): + + id = 'TestDataProviderOne' + + def provide_data(self, **kwargs: Dict[str, Any]) -> Any: + return "data_providers" + + +class TestDataProviderTwo(DataProvider): + + id = 'TestDataProviderTwo' + + def provide_data(self, **kwargs: Dict[str, Any]) -> Any: + return "data_providers" + + +class TestObserver(Observer): + + def __init__(self) -> None: + self.update_count = 0 + + def update(self, observable, **kwargs) -> None: + self.update_count += 1 diff --git a/investing_bot_framework/tests/core/executors/resources.py b/investing_bot_framework/tests/core/executors/resources.py new file mode 100644 index 00000000..3904dbec --- /dev/null +++ b/investing_bot_framework/tests/core/executors/resources.py @@ -0,0 +1,40 @@ +from typing import Dict, Any +from time import sleep +from wrapt import synchronized + +from investing_bot_framework.core.workers import Worker +from investing_bot_framework.core.events.observer import Observer + + +class TestObserver(Observer): + + def __init__(self) -> None: + self.update_count = 0 + + @synchronized + def update(self, observable, **kwargs) -> None: + self.update_count += 1 + + +class TestWorkerOne(Worker): + id = 'TestWorkerOne' + + def work(self, **kwargs: Dict[str, Any]) -> None: + # Simulate some work + sleep(1) + + +class TestWorkerTwo(Worker): + id = 'TestWorkerTwo' + + def work(self, **kwargs: Dict[str, Any]) -> None: + # Simulate some work + sleep(1) + + +class TestWorkerThree(Worker): + id = 'TestWorkerThree' + + def work(self, **kwargs: Dict[str, Any]) -> None: + # Simulate some work + sleep(1) diff --git a/investing_bot_framework/tests/core/executors/test_executor.py b/investing_bot_framework/tests/core/executors/test_executor.py index f4a507c4..7fbc6317 100644 --- a/investing_bot_framework/tests/core/executors/test_executor.py +++ b/investing_bot_framework/tests/core/executors/test_executor.py @@ -1,188 +1,188 @@ -from threading import active_count -from typing import Dict, Any, List -from unittest import TestCase -from time import sleep -from wrapt import synchronized - -from investing_bot_framework.core.workers import Worker -from investing_bot_framework.core.executors import Executor -from investing_bot_framework.core.events.observer import Observer - - -class TestObserver(Observer): - - def __init__(self) -> None: - self.update_count = 0 - - @synchronized - def update(self, observable, **kwargs) -> None: - self.update_count += 1 - - -class TestWorkerOne(Worker): - id = 'TestWorkerOne' - - def work(self, **kwargs: Dict[str, Any]) -> None: - # Simulate some work - sleep(1) - - -class TestWorkerTwo(Worker): - id = 'TestWorkerTwo' - - def work(self, **kwargs: Dict[str, Any]) -> None: - # Simulate some work - sleep(1) - - -class TestWorkerThree(Worker): - id = 'TestWorkerThree' - - def work(self, **kwargs: Dict[str, Any]) -> None: - # Simulate some work - sleep(1) - - -class TestExecutor(Executor): - - def __init__(self, workers: List[Worker] = None): - super(TestExecutor, self).__init__(max_workers=2) - - self._registered_workers = workers - - def create_workers(self) -> List[Worker]: - return self.registered_workers - - @property - def registered_workers(self) -> List[Worker]: - return self._registered_workers - - -class TestStandardExecutor(TestCase): - - def test(self) -> None: - executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo()]) - observer = TestObserver() - executor.add_observer(observer) - - # Make sure the initialization is correct - self.assertEqual(len(executor.registered_workers), 2) - self.assertEqual(active_count(), 1) - - # Start the executor - executor.start() - - # 3 Threads must be running - self.assertTrue(executor.processing) - self.assertEqual(active_count(), 3) - - sleep(2) - - # After finishing only 1 thread must be active - self.assertEqual(active_count(), 1) - self.assertFalse(executor.processing) - - # Observer must have been updated by the executor - self.assertEqual(observer.update_count, 1) - - # Start the executor - executor.start() - - # 3 Threads must be running - self.assertTrue(executor.processing) - self.assertEqual(active_count(), 3) - - sleep(2) - - # After finishing only 1 thread must be active - self.assertEqual(active_count(), 1) - self.assertFalse(executor.processing) - - # Observer must have been updated by the executor - self.assertEqual(observer.update_count, 2) - - executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo(), TestWorkerThree()]) - executor.add_observer(observer) - - # Start the executor - executor.start() - - # 3 Threads must be running - self.assertTrue(executor.processing) - self.assertEqual(active_count(), 3) - - sleep(2) - - # After finishing only two threads must be active (main + last worker, because max workers is 2) - self.assertEqual(active_count(), 2) - self.assertTrue(executor.processing) - - sleep(1) - - # After finishing only 1 thread must be active - self.assertEqual(active_count(), 1) - self.assertFalse(executor.processing) - - # Observer must have been updated by the executor - self.assertEqual(observer.update_count, 3) - - - - - - # def test_execution_executor(): -# logger.info("TEST: test DataProviderExecutor execution") -# -# observer = DummyObserver() -# -# data_provider_one = DummyDataProviderWorker() -# data_provider_three = DummyDataProviderWorker() -# -# executor = DataProviderExecutor( -# [ -# data_provider_one, -# data_provider_three -# ] -# ) -# -# executor.add_observer(observer) -# -# assert active_count() == 1 -# -# -# assert active_count() == 3 -# -# sleep(2) -# -# # Check if the observer is updated by the executor -# assert observer.update_count == 1 -# -# data_provider_one = DummyDataProviderWorker() -# -# executor = DataProviderExecutor( -# [ -# data_provider_one, -# ] -# ) -# -# executor.add_observer(observer) -# -# assert active_count() == 1 -# -# executor.start() -# -# assert active_count() == 2 -# -# sleep(2) -# -# # Check if the observer is updated by the executor -# assert observer.update_count == 2 -# -# executor.start() -# -# sleep(2) -# -# # Check if the observer is updated by the executor -# assert observer.update_count == 3 -# -# logger.info("TEST FINISHED") \ No newline at end of file +# from threading import active_count +# from typing import Dict, Any, List +# from unittest import TestCase +# from time import sleep +# from wrapt import synchronized +# +# from investing_bot_framework.core.workers import Worker +# from investing_bot_framework.core.executors import Executor +# from investing_bot_framework.core.events.observer import Observer +# +# +# class TestObserver(Observer): +# +# def __init__(self) -> None: +# self.update_count = 0 +# +# @synchronized +# def update(self, observable, **kwargs) -> None: +# self.update_count += 1 +# +# +# class TestWorkerOne(Worker): +# id = 'TestWorkerOne' +# +# def work(self, **kwargs: Dict[str, Any]) -> None: +# # Simulate some work +# sleep(1) +# +# +# class TestWorkerTwo(Worker): +# id = 'TestWorkerTwo' +# +# def work(self, **kwargs: Dict[str, Any]) -> None: +# # Simulate some work +# sleep(1) +# +# +# class TestWorkerThree(Worker): +# id = 'TestWorkerThree' +# +# def work(self, **kwargs: Dict[str, Any]) -> None: +# # Simulate some work +# sleep(1) +# +# +# class TestExecutor(Executor): +# +# def __init__(self, workers: List[Worker] = None): +# super(TestExecutor, self).__init__(max_workers=2) +# +# self._registered_workers = workers +# +# def create_workers(self) -> List[Worker]: +# return self.registered_workers +# +# @property +# def registered_workers(self) -> List[Worker]: +# return self._registered_workers +# +# +# class TestStandardExecutor(TestCase): +# +# def test(self) -> None: +# executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo()]) +# observer = TestObserver() +# executor.add_observer(observer) +# +# # Make sure the initialization is correct +# self.assertEqual(len(executor.registered_workers), 2) +# self.assertEqual(active_count(), 1) +# +# # Start the executor +# executor.start() +# +# # 3 Threads must be running +# self.assertTrue(executor.processing) +# self.assertEqual(active_count(), 3) +# +# sleep(2) +# +# # After finishing only 1 thread must be active +# self.assertEqual(active_count(), 1) +# self.assertFalse(executor.processing) +# +# # Observer must have been updated by the executor +# self.assertEqual(observer.update_count, 1) +# +# # Start the executor +# executor.start() +# +# # 3 Threads must be running +# self.assertTrue(executor.processing) +# self.assertEqual(active_count(), 3) +# +# sleep(2) +# +# # After finishing only 1 thread must be active +# self.assertEqual(active_count(), 1) +# self.assertFalse(executor.processing) +# +# # Observer must have been updated by the executor +# self.assertEqual(observer.update_count, 2) +# +# executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo(), TestWorkerThree()]) +# executor.add_observer(observer) +# +# # Start the executor +# executor.start() +# +# # 3 Threads must be running +# self.assertTrue(executor.processing) +# self.assertEqual(active_count(), 3) +# +# sleep(2) +# +# # After finishing only two threads must be active (main + last worker, because max workers is 2) +# self.assertEqual(active_count(), 2) +# self.assertTrue(executor.processing) +# +# sleep(1) +# +# # After finishing only 1 thread must be active +# self.assertEqual(active_count(), 1) +# self.assertFalse(executor.processing) +# +# # Observer must have been updated by the executor +# self.assertEqual(observer.update_count, 3) +# +# +# +# +# +# # def test_execution_executor(): +# # logger.info("TEST: test DataProviderExecutor execution") +# # +# # observer = DummyObserver() +# # +# # data_provider_one = DummyDataProviderWorker() +# # data_provider_three = DummyDataProviderWorker() +# # +# # executor = DataProviderExecutor( +# # [ +# # data_provider_one, +# # data_provider_three +# # ] +# # ) +# # +# # executor.add_observer(observer) +# # +# # assert active_count() == 1 +# # +# # +# # assert active_count() == 3 +# # +# # sleep(2) +# # +# # # Check if the observer is updated by the executor +# # assert observer.update_count == 1 +# # +# # data_provider_one = DummyDataProviderWorker() +# # +# # executor = DataProviderExecutor( +# # [ +# # data_provider_one, +# # ] +# # ) +# # +# # executor.add_observer(observer) +# # +# # assert active_count() == 1 +# # +# # executor.start() +# # +# # assert active_count() == 2 +# # +# # sleep(2) +# # +# # # Check if the observer is updated by the executor +# # assert observer.update_count == 2 +# # +# # executor.start() +# # +# # sleep(2) +# # +# # # Check if the observer is updated by the executor +# # assert observer.update_count == 3 +# # +# # logger.info("TEST FINISHED") \ No newline at end of file diff --git a/investing_bot_framework/tests/core/executors/test_scheduler.py b/investing_bot_framework/tests/core/executors/test_scheduler.py index 2ac02073..ffa2f9ed 100644 --- a/investing_bot_framework/tests/core/executors/test_scheduler.py +++ b/investing_bot_framework/tests/core/executors/test_scheduler.py @@ -1,15 +1,15 @@ import random +import pytest from uuid import uuid4 -from unittest import TestCase from datetime import datetime, timedelta from investing_bot_framework.core.executors.execution_scheduler import ExecutionScheduler, ExecutionTask from investing_bot_framework.core.utils import TimeUnit -class TestExecutionScheduler(TestCase): +class TestExecutionScheduler(object): - def setUp(self) -> None: + def setup_method(self) -> None: self.execution_task_one = { 'execution_id': uuid4().__str__(), @@ -54,17 +54,17 @@ def test(self): # All tasks must be scheduled the first planning planning = scheduler.schedule_executions() - self.assertTrue(self.execution_task_one['execution_id'] in planning) - self.assertTrue(self.execution_task_two['execution_id'] in planning) - self.assertTrue(self.execution_task_three['execution_id'] in planning) - self.assertTrue(self.execution_task_four['execution_id'] in planning) + assert self.execution_task_one['execution_id'] in planning + assert self.execution_task_two['execution_id'] in planning + assert self.execution_task_three['execution_id'] in planning + assert self.execution_task_four['execution_id'] in planning # Only Task 1 must be in the planning planning = scheduler.schedule_executions() - self.assertTrue(self.execution_task_one['execution_id'] in planning) - self.assertFalse(self.execution_task_two['execution_id'] in planning) - self.assertFalse(self.execution_task_three['execution_id'] in planning) - self.assertFalse(self.execution_task_four['execution_id'] in planning) + assert self.execution_task_one['execution_id'] in planning + assert self.execution_task_two['execution_id'] not in planning + assert self.execution_task_three['execution_id'] not in planning + assert self.execution_task_four['execution_id'] not in planning minus_time_delta = datetime.now() - timedelta(seconds=self.execution_task_two['interval']) appointments = scheduler._planning.keys() @@ -79,10 +79,10 @@ def test(self): # Task 1, 2 must be in the planning planning = scheduler.schedule_executions() - self.assertTrue(self.execution_task_one['execution_id'] in planning) - self.assertTrue(self.execution_task_two['execution_id'] in planning) - self.assertFalse(self.execution_task_three['execution_id'] in planning) - self.assertFalse(self.execution_task_four['execution_id'] in planning) + assert self.execution_task_one['execution_id'] in planning + assert self.execution_task_two['execution_id'] in planning + assert self.execution_task_three['execution_id'] not in planning + assert self.execution_task_four['execution_id'] not in planning minus_time_delta = datetime.now() - timedelta(minutes=self.execution_task_three['interval']) appointments = scheduler._planning.keys() @@ -97,10 +97,10 @@ def test(self): # Task 1, 2 and 3 must be in the planning planning = scheduler.schedule_executions() - self.assertTrue(self.execution_task_one['execution_id'] in planning) - self.assertTrue(self.execution_task_two['execution_id'] in planning) - self.assertTrue(self.execution_task_three['execution_id'] in planning) - self.assertFalse(self.execution_task_four['execution_id'] in planning) + assert self.execution_task_one['execution_id'] in planning + assert self.execution_task_two['execution_id'] in planning + assert self.execution_task_three['execution_id'] in planning + assert self.execution_task_four['execution_id'] not in planning minus_time_delta = datetime.now() - timedelta(hours=self.execution_task_four['interval']) appointments = scheduler._planning.keys() @@ -112,33 +112,33 @@ def test(self): scheduler._planning[appointment].interval, last_run=minus_time_delta ) - + # # Task 1, 2 and 3 must be in the planning planning = scheduler.schedule_executions() - self.assertTrue(self.execution_task_one['execution_id'] in planning) - self.assertTrue(self.execution_task_two['execution_id'] in planning) - self.assertTrue(self.execution_task_three['execution_id'] in planning) - self.assertTrue(self.execution_task_four['execution_id'] in planning) + assert self.execution_task_one['execution_id'] in planning + assert self.execution_task_two['execution_id'] in planning + assert self.execution_task_three['execution_id'] in planning + assert self.execution_task_four['execution_id'] in planning def test_exceptions(self) -> None: scheduler = ExecutionScheduler() - with self.assertRaises(Exception) as context: + with pytest.raises(Exception) as e_info: scheduler.add_execution_task(**self.wrong_execution_task_one) - self.assertTrue("Interval for task time unit is smaller then 1" in str(context.exception)) + assert "Interval for task time unit is smaller then 1" in str(e_info.value) - with self.assertRaises(Exception) as context: + with pytest.raises(Exception) as e_info: scheduler.add_execution_task(**self.wrong_execution_task_two) - self.assertTrue("Appoint must set an interval with the corresponding time unit" in str(context.exception)) + assert "Appoint must set an interval with the corresponding time unit" in str(e_info.value) scheduler.add_execution_task(**self.execution_task_one) - with self.assertRaises(Exception) as context: + with pytest.raises(Exception) as e_info: scheduler.add_execution_task(**self.execution_task_one) - self.assertTrue("Can't add execution task, execution id is already taken" in str(context.exception)) + assert "Can't add execution task, execution id is already taken" in str(e_info.value) diff --git a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py index 974d706f..4fac1ad9 100644 --- a/investing_bot_framework/tests/core/resolvers/test_database_resolver.py +++ b/investing_bot_framework/tests/core/resolvers/test_database_resolver.py @@ -1,5 +1,4 @@ import os -from unittest import TestCase from sqlalchemy import Column, String, Integer from investing_bot_framework.tests.resources import BaseTestMixin, utils @@ -12,9 +11,9 @@ class TestModel(db.model): name = Column(String()) -class TestDatabaseResolverConfiguration(TestCase, BaseTestMixin): +class TestDatabaseResolverConfiguration(BaseTestMixin): - def setUp(self) -> None: + def setup_method(self) -> None: self.initialize_environment() def test_configuration(self): @@ -22,21 +21,21 @@ def test_configuration(self): db.configure() # Check if all properties are configured - self.assertIsNotNone(db.Session) - self.assertIsNotNone(db.engine) - self.assertIsNotNone(db.session_factory) - self.assertIsNotNone(db.database_path) - self.assertTrue(os.path.isfile(db.database_path)) + assert db.Session is not None + assert db.engine is not None + assert db.session_factory is not None + assert db.database_path is not None + assert os.path.isfile(db.database_path) == True - def tearDown(self) -> None: + def teardown_method(self) -> None: if os.path.isfile(db.database_path): os.remove(db.database_path) -class TestDatabaseResolverModel(TestCase, BaseTestMixin): +class TestDatabaseResolverModel(BaseTestMixin): - def setUp(self) -> None: + def setup_method(self) -> None: self.initialize_environment() settings.configure() db.configure() @@ -47,14 +46,14 @@ def test(self) -> None: model = TestModel(name=utils.random_string(10)) model.save() db.session.commit() - self.assertEqual(1, len(TestModel.query.all())) + assert 1 == len(TestModel.query.all()) model = TestModel(name=utils.random_string(10)) model.save() db.session.commit() - self.assertEqual(2, len(TestModel.query.all())) + assert 2 == len(TestModel.query.all()) - def tearDown(self) -> None: + def teardown_method(self) -> None: if os.path.isfile(db.database_path): os.remove(db.database_path) From 67af58b140ce6266861d7bc17bca91ec88e67ac3 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 11:49:11 +0200 Subject: [PATCH 51/55] Add pytest framework --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c384b3bd..dc54f685 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ pandas==0.25.3 wrapt==1.11.2 colorama==0.4.3 SQLAlchemy==1.3.13 - +pytest==5.4.3 From d955a394b1799d70c174f8ee256d56cc6abf8ff5 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 12:05:50 +0200 Subject: [PATCH 52/55] Add pytest framework --- .../tests/core/executors/resources.py | 18 +- .../tests/core/executors/test_executor.py | 263 +++++------------- 2 files changed, 92 insertions(+), 189 deletions(-) diff --git a/investing_bot_framework/tests/core/executors/resources.py b/investing_bot_framework/tests/core/executors/resources.py index 3904dbec..522f6b42 100644 --- a/investing_bot_framework/tests/core/executors/resources.py +++ b/investing_bot_framework/tests/core/executors/resources.py @@ -1,9 +1,10 @@ -from typing import Dict, Any +from typing import Dict, Any, List from time import sleep from wrapt import synchronized from investing_bot_framework.core.workers import Worker from investing_bot_framework.core.events.observer import Observer +from investing_bot_framework.core.executors import Executor class TestObserver(Observer): @@ -38,3 +39,18 @@ class TestWorkerThree(Worker): def work(self, **kwargs: Dict[str, Any]) -> None: # Simulate some work sleep(1) + + +class TestExecutor(Executor): + + def __init__(self, workers: List[Worker] = None): + super(TestExecutor, self).__init__(max_workers=2) + + self._registered_workers = workers + + def create_workers(self) -> List[Worker]: + return self.registered_workers + + @property + def registered_workers(self) -> List[Worker]: + return self._registered_workers diff --git a/investing_bot_framework/tests/core/executors/test_executor.py b/investing_bot_framework/tests/core/executors/test_executor.py index 7fbc6317..72f2854a 100644 --- a/investing_bot_framework/tests/core/executors/test_executor.py +++ b/investing_bot_framework/tests/core/executors/test_executor.py @@ -1,188 +1,75 @@ -# from threading import active_count -# from typing import Dict, Any, List -# from unittest import TestCase -# from time import sleep -# from wrapt import synchronized -# -# from investing_bot_framework.core.workers import Worker -# from investing_bot_framework.core.executors import Executor -# from investing_bot_framework.core.events.observer import Observer -# -# -# class TestObserver(Observer): -# -# def __init__(self) -> None: -# self.update_count = 0 -# -# @synchronized -# def update(self, observable, **kwargs) -> None: -# self.update_count += 1 -# -# -# class TestWorkerOne(Worker): -# id = 'TestWorkerOne' -# -# def work(self, **kwargs: Dict[str, Any]) -> None: -# # Simulate some work -# sleep(1) -# -# -# class TestWorkerTwo(Worker): -# id = 'TestWorkerTwo' -# -# def work(self, **kwargs: Dict[str, Any]) -> None: -# # Simulate some work -# sleep(1) -# -# -# class TestWorkerThree(Worker): -# id = 'TestWorkerThree' -# -# def work(self, **kwargs: Dict[str, Any]) -> None: -# # Simulate some work -# sleep(1) -# -# -# class TestExecutor(Executor): -# -# def __init__(self, workers: List[Worker] = None): -# super(TestExecutor, self).__init__(max_workers=2) -# -# self._registered_workers = workers -# -# def create_workers(self) -> List[Worker]: -# return self.registered_workers -# -# @property -# def registered_workers(self) -> List[Worker]: -# return self._registered_workers -# -# -# class TestStandardExecutor(TestCase): -# -# def test(self) -> None: -# executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo()]) -# observer = TestObserver() -# executor.add_observer(observer) -# -# # Make sure the initialization is correct -# self.assertEqual(len(executor.registered_workers), 2) -# self.assertEqual(active_count(), 1) -# -# # Start the executor -# executor.start() -# -# # 3 Threads must be running -# self.assertTrue(executor.processing) -# self.assertEqual(active_count(), 3) -# -# sleep(2) -# -# # After finishing only 1 thread must be active -# self.assertEqual(active_count(), 1) -# self.assertFalse(executor.processing) -# -# # Observer must have been updated by the executor -# self.assertEqual(observer.update_count, 1) -# -# # Start the executor -# executor.start() -# -# # 3 Threads must be running -# self.assertTrue(executor.processing) -# self.assertEqual(active_count(), 3) -# -# sleep(2) -# -# # After finishing only 1 thread must be active -# self.assertEqual(active_count(), 1) -# self.assertFalse(executor.processing) -# -# # Observer must have been updated by the executor -# self.assertEqual(observer.update_count, 2) -# -# executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo(), TestWorkerThree()]) -# executor.add_observer(observer) -# -# # Start the executor -# executor.start() -# -# # 3 Threads must be running -# self.assertTrue(executor.processing) -# self.assertEqual(active_count(), 3) -# -# sleep(2) -# -# # After finishing only two threads must be active (main + last worker, because max workers is 2) -# self.assertEqual(active_count(), 2) -# self.assertTrue(executor.processing) -# -# sleep(1) -# -# # After finishing only 1 thread must be active -# self.assertEqual(active_count(), 1) -# self.assertFalse(executor.processing) -# -# # Observer must have been updated by the executor -# self.assertEqual(observer.update_count, 3) -# -# -# -# -# -# # def test_execution_executor(): -# # logger.info("TEST: test DataProviderExecutor execution") -# # -# # observer = DummyObserver() -# # -# # data_provider_one = DummyDataProviderWorker() -# # data_provider_three = DummyDataProviderWorker() -# # -# # executor = DataProviderExecutor( -# # [ -# # data_provider_one, -# # data_provider_three -# # ] -# # ) -# # -# # executor.add_observer(observer) -# # -# # assert active_count() == 1 -# # -# # -# # assert active_count() == 3 -# # -# # sleep(2) -# # -# # # Check if the observer is updated by the executor -# # assert observer.update_count == 1 -# # -# # data_provider_one = DummyDataProviderWorker() -# # -# # executor = DataProviderExecutor( -# # [ -# # data_provider_one, -# # ] -# # ) -# # -# # executor.add_observer(observer) -# # -# # assert active_count() == 1 -# # -# # executor.start() -# # -# # assert active_count() == 2 -# # -# # sleep(2) -# # -# # # Check if the observer is updated by the executor -# # assert observer.update_count == 2 -# # -# # executor.start() -# # -# # sleep(2) -# # -# # # Check if the observer is updated by the executor -# # assert observer.update_count == 3 -# # -# # logger.info("TEST FINISHED") \ No newline at end of file +from threading import active_count +from unittest import TestCase +from time import sleep + +from investing_bot_framework.tests.core.executors.resources import TestExecutor, TestWorkerOne, TestWorkerTwo, \ + TestObserver, TestWorkerThree + + +class TestStandardExecutor: + + def test(self) -> None: + executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo()]) + observer = TestObserver() + executor.add_observer(observer) + + # Make sure the initialization is correct + assert len(executor.registered_workers) == 2 + assert active_count() == 1 + + # Start the executor + executor.start() + + # 3 Threads must be running + assert executor.processing + assert active_count() == 3 + + sleep(2) + + # # After finishing only 1 thread must be active + assert active_count(), 1 + assert not executor.processing + + # Observer must have been updated by the executor + assert observer.update_count == 1 + + # Start the executor + executor.start() + + # 3 Threads must be running + assert executor.processing + assert active_count() == 3 + + sleep(2) + + # After finishing only 1 thread must be active + assert active_count() == 1 + assert not executor.processing + + # Observer must have been updated by the executor + assert observer.update_count == 2 + + executor = TestExecutor(workers=[TestWorkerOne(), TestWorkerTwo(), TestWorkerThree()]) + executor.add_observer(observer) + + # Start the executor + executor.start() + + # 3 Threads must be running + assert executor.processing + assert active_count() == 3 + + sleep(2) + + # After finishing only two threads must be active (main + last worker, because max workers is 2) + assert active_count() == 2 + assert executor.processing + + sleep(1) + + # After finishing only 1 thread must be active + assert active_count(), 1 + assert not executor.processing + + # Observer must have been updated by the executor + assert observer.update_count == 3 From b0b2723ebe5c3053376045bcc614b0a16c6185ee Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 12:07:47 +0200 Subject: [PATCH 53/55] Add codecov --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index b8f22ae5..be245b24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,3 +4,6 @@ install: - pip install -r requirements.txt script: pytest # run test + +after_success: + - codecov # submit coverage \ No newline at end of file From 4c800f69bd6e5949960bdebef07e8e978557a4e6 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 12:22:26 +0200 Subject: [PATCH 54/55] Update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 907e57d2..88d45387 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -# Investing Algorithm Framework +[![Build Status](https://travis-ci.org/investingbots/investing-bot-framework.svg?branch=master)](https://travis-ci.org/investingbots/investing-bot-framework) + +# Investing Algorithm Framework The Investing Algorithm Framework is a free and open source Python framework that encourages rapid development and clean, pragmatic design. From 449352f6d40d17ac565cf0fc628ddaa50ef71545 Mon Sep 17 00:00:00 2001 From: DUYN Date: Fri, 19 Jun 2020 12:27:46 +0200 Subject: [PATCH 55/55] remove unused files --- Remove_states_from_context.patch | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 Remove_states_from_context.patch diff --git a/Remove_states_from_context.patch b/Remove_states_from_context.patch deleted file mode 100644 index ee5a184b..00000000 --- a/Remove_states_from_context.patch +++ /dev/null @@ -1,6 +0,0 @@ -Index: investing_bot_framework/core/context/states/__init__.py -=================================================================== ---- investing_bot_framework/core/context/states/__init__.py (revision 59f4076a0c21098a23a0bdef6718b8838b45550b) -+++ investing_bot_framework/core/context/states/__init__.py (revision 59f4076a0c21098a23a0bdef6718b8838b45550b) -@@ -1,1 +0,0 @@ --from investing_bot_framework.core.context.states.bot_state import BotState