From 205e63b1a030432d875bffdde0acfe7013d1c900 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 14:09:38 +0100 Subject: [PATCH 01/52] Removed all modes of evaluating a dependency graph that involve moving numpy arrays across a process boundary. --- quantdsl/application/main.py | 32 ----- quantdsl/application/with_cassandra.py | 13 -- quantdsl/application/with_greenthreads.py | 39 ------ .../with_greenthreads_and_python_objects.py | 7 - quantdsl/application/with_multiprocessing.py | 43 ------ .../with_multiprocessing_and_cassandra.py | 9 -- .../with_multiprocessing_and_sqlalchemy.py | 10 -- quantdsl/application/with_sqlalchemy.py | 7 - quantdsl/infrastructure/runners/__init__.py | 1 - quantdsl/infrastructure/runners/base.py | 125 ------------------ .../infrastructure/runners/distributed.py | 67 ---------- .../infrastructure/runners/multiprocess.py | 125 ------------------ .../infrastructure/runners/singlethread.py | 47 ------- ...ion_with_greenthreads_and_pythonobjects.py | 14 -- ...tion_with_multiprocessing_and_cassandra.py | 37 ------ ...ion_with_multiprocessing_and_sqlalchemy.py | 29 ---- ...cation_with_singlethread_and_sqlalchemy.py | 8 -- ...est_distributed_dependency_graph_runner.py | 111 ---------------- 18 files changed, 724 deletions(-) delete mode 100644 quantdsl/application/main.py delete mode 100644 quantdsl/application/with_cassandra.py delete mode 100644 quantdsl/application/with_greenthreads.py delete mode 100644 quantdsl/application/with_greenthreads_and_python_objects.py delete mode 100644 quantdsl/application/with_multiprocessing.py delete mode 100644 quantdsl/application/with_multiprocessing_and_cassandra.py delete mode 100644 quantdsl/application/with_multiprocessing_and_sqlalchemy.py delete mode 100644 quantdsl/application/with_sqlalchemy.py delete mode 100644 quantdsl/infrastructure/runners/__init__.py delete mode 100644 quantdsl/infrastructure/runners/base.py delete mode 100644 quantdsl/infrastructure/runners/distributed.py delete mode 100644 quantdsl/infrastructure/runners/multiprocess.py delete mode 100644 quantdsl/infrastructure/runners/singlethread.py delete mode 100644 quantdsl/tests/test_application_with_greenthreads_and_pythonobjects.py delete mode 100644 quantdsl/tests/test_application_with_multiprocessing_and_cassandra.py delete mode 100644 quantdsl/tests/test_application_with_multiprocessing_and_sqlalchemy.py delete mode 100644 quantdsl/tests/test_application_with_singlethread_and_sqlalchemy.py delete mode 100644 quantdsl/tests/test_distributed_dependency_graph_runner.py diff --git a/quantdsl/application/main.py b/quantdsl/application/main.py deleted file mode 100644 index 1e4b963..0000000 --- a/quantdsl/application/main.py +++ /dev/null @@ -1,32 +0,0 @@ -import os - - - -def get_quantdsl_app(backend_name='pythonobjects', call_evaluation_queue=None): - # Todo: Put this config stuff under test. - backend = os.environ.get('QUANTDSL_BACKEND', backend_name).strip().lower() - if backend == 'pythonobjects': - from quantdsl.application.with_pythonobjects import QuantDslApplicationWithPythonObjects - quantdsl_app = QuantDslApplicationWithPythonObjects(call_evaluation_queue=call_evaluation_queue) - elif backend == 'sqlalchemy': - from quantdsl.application.with_sqlalchemy import QuantDslApplicationWithSQLAlchemy - db_uri = os.environ.get('QUANTDSL_DB_URI', 'sqlite:////tmp/quantdsl-tmp.db') - quantdsl_app = QuantDslApplicationWithSQLAlchemy(db_uri=db_uri, call_evaluation_queue=call_evaluation_queue) - elif backend == 'cassandra': - from quantdsl.application.with_cassandra import DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE - from quantdsl.application.with_cassandra import QuantDslApplicationWithCassandra - # hosts = [i.strip() for i in os.environ.get('QUANT_DSL_CASSANDRA_HOSTS', 'localhost').split(',')] - keyspace = os.environ.get('QUANTDSL_CASSANDRA_KEYSPACE', DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE).strip() - # port = int(os.environ.get('QUANTDSL_CASSANDRA_PORT', '9042').strip()) - # username = os.environ.get('QUANTDSL_CASSANDRA_USERNAME', '').strip() or None - # password = os.environ.get('QUANTDSL_CASSANDRA_PASSWORD', '').strip() or None - quantdsl_app = QuantDslApplicationWithCassandra( - default_keyspace=keyspace, - # port=port, - # username=username, - # password=password, - call_evaluation_queue=call_evaluation_queue - ) - else: - raise ValueError("Invalid backend ('sqlalchemy', 'cassandra' and 'pythonobjects' are supported): " + backend) - return quantdsl_app \ No newline at end of file diff --git a/quantdsl/application/with_cassandra.py b/quantdsl/application/with_cassandra.py deleted file mode 100644 index b80dbb6..0000000 --- a/quantdsl/application/with_cassandra.py +++ /dev/null @@ -1,13 +0,0 @@ -from eventsourcing.application.with_cassandra import EventSourcingWithCassandra -from quantdsl.application.base import QuantDslApplication - - -DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE = 'quantdsl' - - -class QuantDslApplicationWithCassandra(EventSourcingWithCassandra, QuantDslApplication): - - def __init__(self, default_keyspace=DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE, *args, **kwargs): - super(QuantDslApplicationWithCassandra, self).__init__(default_keyspace=default_keyspace, *args, **kwargs) - - diff --git a/quantdsl/application/with_greenthreads.py b/quantdsl/application/with_greenthreads.py deleted file mode 100644 index e8f29f6..0000000 --- a/quantdsl/application/with_greenthreads.py +++ /dev/null @@ -1,39 +0,0 @@ -import multiprocessing -import gevent -# from gevent import monkey -# monkey.patch_all(thread=False, socket=False) -import gevent.pool -from gevent.queue import JoinableQueue - -from quantdsl.application.base import QuantDslApplication - - -class QuantDslApplicationWithGreenThreads(QuantDslApplication): - - def __init__(self, num_workers=100, *args, **kwargs): - result_counters = {} - usage_counters = {} - call_evaluation_queue = JoinableQueue() - - super(QuantDslApplicationWithGreenThreads, self).__init__(call_evaluation_queue=call_evaluation_queue, - result_counters=result_counters, - usage_counters=usage_counters, *args, **kwargs) - - call_result_lock = None - self.compute_pool = multiprocessing.Pool(multiprocessing.cpu_count()) - - # Start evaluation worker threads. - self.workers = [] - self.pool = gevent.pool.Pool(num_workers) - for _ in range(num_workers): - self.pool.apply_async( - func=self.loop_on_evaluation_queue, - args=(call_result_lock, self.compute_pool, result_counters, usage_counters), - ) - - def close(self): - super(QuantDslApplicationWithGreenThreads, self).close() - try: - self.compute_pool.terminate() - finally: - self.pool.kill(timeout=1) diff --git a/quantdsl/application/with_greenthreads_and_python_objects.py b/quantdsl/application/with_greenthreads_and_python_objects.py deleted file mode 100644 index 4ea53cc..0000000 --- a/quantdsl/application/with_greenthreads_and_python_objects.py +++ /dev/null @@ -1,7 +0,0 @@ -from quantdsl.application.with_greenthreads import QuantDslApplicationWithGreenThreads -from quantdsl.application.with_pythonobjects import QuantDslApplicationWithPythonObjects - - -class QuantDslApplicationWithGreenThreadsAndPythonObjects(QuantDslApplicationWithGreenThreads, - QuantDslApplicationWithPythonObjects): - pass \ No newline at end of file diff --git a/quantdsl/application/with_multiprocessing.py b/quantdsl/application/with_multiprocessing.py deleted file mode 100644 index 6e5b959..0000000 --- a/quantdsl/application/with_multiprocessing.py +++ /dev/null @@ -1,43 +0,0 @@ -from multiprocessing.pool import Pool -from multiprocessing import Manager - -from quantdsl.application.base import QuantDslApplication - - -class QuantDslApplicationWithMultiprocessing(QuantDslApplication): - - def __init__(self, num_workers=None, call_evaluation_queue=None, **kwargs): - if num_workers is not None: - assert call_evaluation_queue is None - # Parent. - self.pool = Pool(processes=num_workers) - self.manager = Manager() - self.call_evaluation_queue = self.manager.Queue() - - else: - # Child. - self.pool = None - super(QuantDslApplicationWithMultiprocessing, self).__init__(call_evaluation_queue=call_evaluation_queue, **kwargs) - - if self.pool: - # Start worker pool. - app_kwargs = self.get_subprocess_application_args() - args = (self.manager.Lock(), self.__class__, app_kwargs) - for i in range(num_workers): - self.pool.apply_async(loop_on_evaluation_queue, args) - - def get_subprocess_application_args(self): - app_kwargs = dict( - call_evaluation_queue=self.call_evaluation_queue, - ) - return app_kwargs - - def close(self): - super(QuantDslApplicationWithMultiprocessing, self).close() - if self.pool: - self.pool.terminate() - - -def loop_on_evaluation_queue(call_result_lock, application_cls, app_kwargs): - app = application_cls(**app_kwargs) - app.loop_on_evaluation_queue(call_result_lock=call_result_lock) diff --git a/quantdsl/application/with_multiprocessing_and_cassandra.py b/quantdsl/application/with_multiprocessing_and_cassandra.py deleted file mode 100644 index 9a724c4..0000000 --- a/quantdsl/application/with_multiprocessing_and_cassandra.py +++ /dev/null @@ -1,9 +0,0 @@ -from quantdsl.application.with_cassandra import QuantDslApplicationWithCassandra -from quantdsl.application.with_multiprocessing import QuantDslApplicationWithMultiprocessing - - -class QuantDslApplicationWithMultiprocessingAndCassandra(QuantDslApplicationWithMultiprocessing, - QuantDslApplicationWithCassandra): - - pass - diff --git a/quantdsl/application/with_multiprocessing_and_sqlalchemy.py b/quantdsl/application/with_multiprocessing_and_sqlalchemy.py deleted file mode 100644 index f245ac1..0000000 --- a/quantdsl/application/with_multiprocessing_and_sqlalchemy.py +++ /dev/null @@ -1,10 +0,0 @@ -from quantdsl.application.with_sqlalchemy import QuantDslApplicationWithSQLAlchemy -from quantdsl.application.with_multiprocessing import QuantDslApplicationWithMultiprocessing - - -class QuantDslApplicationWithMultiprocessingAndSQLAlchemy(QuantDslApplicationWithMultiprocessing, - QuantDslApplicationWithSQLAlchemy): - def get_subprocess_application_args(self): - kwargs = super(QuantDslApplicationWithMultiprocessingAndSQLAlchemy, self).get_subprocess_application_args() - kwargs['db_uri'] = self.db_uri - return kwargs \ No newline at end of file diff --git a/quantdsl/application/with_sqlalchemy.py b/quantdsl/application/with_sqlalchemy.py deleted file mode 100644 index c928055..0000000 --- a/quantdsl/application/with_sqlalchemy.py +++ /dev/null @@ -1,7 +0,0 @@ -from eventsourcing.application.with_sqlalchemy import EventSourcingWithSQLAlchemy -from quantdsl.application.base import QuantDslApplication - - -class QuantDslApplicationWithSQLAlchemy(EventSourcingWithSQLAlchemy, QuantDslApplication): - - pass \ No newline at end of file diff --git a/quantdsl/infrastructure/runners/__init__.py b/quantdsl/infrastructure/runners/__init__.py deleted file mode 100644 index 6ccd6fe..0000000 --- a/quantdsl/infrastructure/runners/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'john' diff --git a/quantdsl/infrastructure/runners/base.py b/quantdsl/infrastructure/runners/base.py deleted file mode 100644 index b743347..0000000 --- a/quantdsl/infrastructure/runners/base.py +++ /dev/null @@ -1,125 +0,0 @@ -# from abc import ABCMeta, abstractmethod -# -# import six -# -# from quantdsl.domain.model.call_specification import CallSpecification -# from quantdsl.domain.model.dependency_graph import DependencyGraph -# from quantdsl.exceptions import DslSyntaxError, DslSystemError -# from quantdsl.semantics import Module, DslNamespace, DslExpression, compile_dsl_module, list_fixing_dates - - -# class DependencyGraphRunner(six.with_metaclass(ABCMeta)): -# -# def __init__(self, dependency_graph): -# assert isinstance(dependency_graph, DependencyGraph) -# self.dependency_graph = dependency_graph -# -# def evaluate(self, **kwds): -# self.run(**kwds) -# try: -# return self.results_repo[self.dependency_graph.root_stub_id] -# except KeyError: -# raise DslSystemError("Result not found for root stub ID '{}'.".format( -# self.dependency_graph.root_stub_id -# )) -# -# @abstractmethod -# def run(self, **kwargs): -# self.run_kwds = kwargs -# self.call_count = 0 -# self.results_repo = {} -# self.dependencies = {} -# -# def get_evaluation_kwds(self, dsl_source, effective_present_time): -# evaluation_kwds = self.run_kwds.copy() -# -# from quantdsl.services import list_fixing_dates -# from quantdsl.domain.services.parser import dsl_parse -# stubbed_module = dsl_parse(dsl_source) -# assert isinstance(stubbed_module, Module) -# fixing_dates = list_fixing_dates(stubbed_module) -# if effective_present_time is not None: -# fixing_dates.append(effective_present_time) -# -# # Rebuild the data structure (there was a problem, but I can't remember what it was. -# # Todo: Try without this block, perhaps the problem doesn't exist anymore. -# if 'all_market_prices' in evaluation_kwds: -# all_market_prices = evaluation_kwds.pop('all_market_prices') -# evaluation_kwds['all_market_prices'] = dict() -# for market_name in all_market_prices.keys(): -# # if market_name not in market_names: -# # continue -# market_prices = dict() -# for date in fixing_dates: -# market_prices[date] = all_market_prices[market_name][date] -# evaluation_kwds['all_market_prices'][market_name] = market_prices -# return evaluation_kwds - - -# def evaluate_call(call_spec, register_call_result): -# """ -# Evaluates the stubbed expr identified by 'call_requirement_id'. -# """ -# assert isinstance(call_spec, CallSpecification) -# -# evaluation_kwds = call_spec.evaluation_kwds.copy() -# -# # If this call has an effective present time value, use it as the 'present_time' in the evaluation_kwds. -# # This results from e.g. the Wait DSL element. Calls near the root of the expression might not have an -# # effective present time value, and the present time will be the observation time of the evaluation. -# -# # Evaluate the stubbed expr str. -# # - parse the expr -# try: -# # Todo: Rework this dependency. Figure out how to use alternative set of DSL classes when multiprocessing. -# from quantdsl.domain.services.parser import dsl_parse -# stubbed_module = dsl_parse(call_spec.dsl_expr_str) -# except DslSyntaxError: -# raise -# -# assert isinstance(stubbed_module, Module), "Parsed stubbed expr string is not a module: %s" % stubbed_module -# -# # - build a namespace from the dependency values -# dsl_locals = DslNamespace(call_spec.dependency_values) -# -# # - compile the parsed expr -# dsl_expr = stubbed_module.body[0].reduce(dsl_locals=dsl_locals, dsl_globals=DslNamespace()) -# assert isinstance(dsl_expr, DslExpression), dsl_expr -# -# # - evaluate the compiled expr -# result_value = dsl_expr.evaluate(**evaluation_kwds) -# -# # - store the result -# register_call_result(call_id=call_spec.id, result_value=result_value) - - -# def handle_result(call_requirement_id, result_value, results, dependents, dependencies, execution_queue): -# -# # Set the results. -# results[call_requirement_id] = result_value -# -# # Check if dependents are ready to be executed. -# for dependent_id in dependents[call_requirement_id]: -# if dependent_id in results: -# continue -# subscriber_required_ids = dependencies[dependent_id] -# # It's ready unless it requires a call that doesn't have a result yet. -# for required_id in subscriber_required_ids: -# # - don't need to see if this call has a result, that's why we're here! -# if required_id != call_requirement_id: -# # - check if the required call already has a result -# if required_id not in results: -# break -# else: -# # All required results exist for the dependent call. -# execution_queue.put(dependent_id) -# -# # Check for results that should be deleted. -# # - dependency results should be deleted if there is a result for each dependent of the dependency -# for dependency_id in dependencies[call_requirement_id]: -# for dependent_id in dependents[dependency_id]: -# if dependent_id != call_requirement_id and dependent_id not in results: -# # Need to keep it. -# break -# else: -# del(results[dependency_id]) diff --git a/quantdsl/infrastructure/runners/distributed.py b/quantdsl/infrastructure/runners/distributed.py deleted file mode 100644 index 66f9e5c..0000000 --- a/quantdsl/infrastructure/runners/distributed.py +++ /dev/null @@ -1,67 +0,0 @@ -# from quantdsl.application.base import QuantDslApplication -# from quantdsl.infrastructure.celery.tasks import celery_evaluate_call -# from quantdsl.infrastructure.runners.base import DependencyGraphRunner -# -# # We can either run the leaf nodes and run dependents that are ready to run, -# # or we can reverse the result of topological sort, and run through a series. -# # Running leaves first incurs the cost of checking for the existence of a result -# # for each of the dependencies of each potentially runnable dependent. It allows for -# # parallel evaluation of the nodes, which would allow all runnable nodes to be run -# # at the same time. But if each node is executed on a different machine, each would -# # require both the results it depends on and the simulation (e.g. market simulation) -# # it depends on. That means the cost of checking for runnable nodes would be added -# # to the cost of transferring the simulation and results data. If everything was on the -# # same machine, there wouldn't be network transfers. -# # Running in reverse topological order would avoid dependency checking, all data -# # would be local, but dependencies could not be evaluated in parallel, because we -# # wouldn't know when to queue the nodes, and queuing them too early (before dependencies -# # evaluated) would fail. -# # If the dependency graph is obtained before it is persisted, the size of the computation -# # will be limited to the size of the dependency graph that will fit in memory. However -# # if the dependency graph is persisted as it is generated - by refining the generation -# # of the dependency graph to separate generating nodes from storing nodes - then computations -# # will be limited to the size of the storage ("infinite"). -# # In the case where the dependency graph is persisted as it is generated, then evaluation -# # of the nodes will incur the cost of retrieving the nodes. -# # Similarly for the (e.g. market) simulation: if we are to avoid constraining the size of -# # the simulation to the memory rather than the storage, the simulation will need to be -# # stored, and the cost of retrieving the simulation from storage will be incurred. -# # In other words, to avoid limiting compuation to the size of memory, both the dependency -# # graph (and any simulation it uses) will need to be stored, and therefore retrieved. -# # Hence, by comparison with evaluation of the dependency graph in series, parallel -# # computation incurs only the extra cost of working out which nodes are ready to run, -# # so long as we identify that the cost of storing and retrieving the nodes and the -# # simulation are associated with avoiding constraining the computation by available memory. -# -# # So the alternatives are: -# # 1. either use reverse topological order OR discover runnable nodes after each node finishes -# # 2. either put dependency graph nodes etc. in memory OR in a persistence mechanism -# # 3. either run in local process with Python queues OR use a message queue and workers -# -# # Because using message queues and workers depends on persisting the data, there are 3 choices: -# # 1. in memory data + local processing (serialized or in parallel) -# # 2. persisted data + local processing (serialized or in parallel) -# # 3. persisted data + distributed processing (serialized or in parallel) -# # -# # If the data is in memory, the generation of the dependency graph and the simulation need -# # to be done just before the evaluation. If the data is persisted, then generating the call -# # dependency graph from the DSL expression could also be distributed. -# # -# # In case we want to avoid limiting the computation to the memory of a single machine, we -# # will need to persist the call dependency graph, the simulation, and the results. -# -# class DistributedDependencyGraphRunner(DependencyGraphRunner): -# -# def __init__(self, dependency_graph, app): -# super(DistributedDependencyGraphRunner, self).__init__(dependency_graph) -# assert isinstance(app, QuantDslApplication) -# self.app = app -# -# def run(self, **kwargs): -# super(DistributedDependencyGraphRunner, self).run(**kwargs) -# -# self.app.register_dependency_graph(self.dependency_graph) -# -# # Enqueue an execution job for each leaf of the dependency graph. -# for call_id in self.dependency_graph.leaf_ids: -# celery_evaluate_call(call_id, evaluation_kwds=kwargs) diff --git a/quantdsl/infrastructure/runners/multiprocess.py b/quantdsl/infrastructure/runners/multiprocess.py deleted file mode 100644 index c49464b..0000000 --- a/quantdsl/infrastructure/runners/multiprocess.py +++ /dev/null @@ -1,125 +0,0 @@ -# from threading import Thread -# from time import sleep -# from quantdsl.domain.model.call_specification import CallSpecification -# -# from quantdsl.infrastructure.runners.base import DependencyGraphRunner, evaluate_call, handle_result -# from quantdsl.domain.services.dependency_graphs import get_dependency_values -# from quantdsl.domain.model.call_requirement import StubbedCall -# -# -# class MultiProcessingDependencyGraphRunner(DependencyGraphRunner): -# -# def __init__(self, dependency_graph, pool_size=None): -# super(MultiProcessingDependencyGraphRunner, self).__init__(dependency_graph) -# self.pool_size = pool_size -# -# def run(self, **kwargs): -# super(MultiProcessingDependencyGraphRunner, self).run(**kwargs) -# import multiprocessing -# # Set up pool of evaluation workers. -# self.evaluation_pool = multiprocessing.Pool(processes=self.pool_size) -# # Set up a 'shared memory' dependency graph. -# self.manager = multiprocessing.Manager() -# self.execution_queue = self.manager.Queue() -# self.result_queue = self.manager.Queue() -# self.results_repo = self.manager.dict() -# self.calls_dict = self.manager.dict() -# self.calls_dict.update(self.dependency_graph.call_requirements) -# self.requirements = self.manager.dict() -# self.requirements.update(self.dependency_graph.requirements) -# self.dependents = self.manager.dict() -# self.dependents.update(self.dependency_graph.dependents) -# self.errors = [] -# # if 'all_market_prices' in self.run_kwds: -# # all_market_prices = self.run_kwds.pop('all_market_prices') -# # all_market_prices_dict = self.manager.dict() -# # for market_name, market_prices in all_market_prices.items(): -# # all_market_prices_dict[market_name] = market_prices.copy() -# # # market_prices_dict = self.manager.dict() -# # # all_market_prices_dict[market_name] = market_prices_dict -# # # for fixing_date, market_price in market_prices.items(): -# # # market_prices_dict[fixing_date] = market_price -# # self.run_kwds['all_market_prices'] = all_market_prices_dict -# -# evaluation_thread = Thread(target=self.evaluate_calls) -# evaluation_thread.daemon = True -# evaluation_thread.start() -# -# results_thread = Thread(target=self.handle_results) -# results_thread.daemon = True -# results_thread.start() -# -# # Put the leaves on the execution queue. -# for call_requirement_id in self.dependency_graph.leaf_ids: -# self.execution_queue.put(call_requirement_id) -# -# while results_thread.isAlive() and evaluation_thread.isAlive(): -# sleep(1) -# if evaluation_thread.isAlive(): -# self.execution_queue.put(None) -# evaluation_thread.join() -# if results_thread.isAlive(): -# self.result_queue.put(None) -# results_thread.join() -# -# if self.errors: -# raise self.errors[0] -# -# def evaluate_calls(self): -# try: -# while True: -# call_requirement_id = self.execution_queue.get() -# if call_requirement_id is None: -# break -# else: -# call_requirement = self.calls_dict[call_requirement_id] -# assert isinstance(call_requirement, StubbedCall) -# dsl_source, effective_present_time = call_requirement -# evaluation_kwds = self.get_evaluation_kwds(dsl_source, effective_present_time) -# dependency_values = get_dependency_values(call_requirement_id, self.requirements, self.results_repo) -# -# call_spec = CallSpecification( -# id=call_requirement_id, -# dsl_expr_str=dsl_source, -# effective_present_time=effective_present_time, -# evaluation_kwds=evaluation_kwds, -# dependency_values=dependency_values, -# ) -# -# def target(): -# async_result = self.evaluation_pool.apply_async(evaluate_call, ( -# call_spec, -# self.result_queue, -# )) -# try: -# async_result.get() -# except Exception as error: -# self.result_queue.put(None) -# self.execution_queue.put(None) -# self.errors.append(error) -# thread = Thread(target=target) -# thread.daemon = True -# thread.start() -# -# self.call_count += 1 -# except: -# self.result_queue.put(None) -# raise -# -# def handle_results(self): -# try: -# while True: -# result = self.result_queue.get() -# if result is None: -# break -# else: -# (call_requirement_id, result_value) = result -# handle_result(call_requirement_id, result_value, self.results_repo, self.dependents, -# self.requirements, self.execution_queue) -# -# if call_requirement_id == self.dependency_graph.root_stub_id: -# break -# except Exception as error: -# self.execution_queue.put(None) -# self.errors.append(error) -# raise diff --git a/quantdsl/infrastructure/runners/singlethread.py b/quantdsl/infrastructure/runners/singlethread.py deleted file mode 100644 index 1c47b15..0000000 --- a/quantdsl/infrastructure/runners/singlethread.py +++ /dev/null @@ -1,47 +0,0 @@ -# import six -# -# from quantdsl.infrastructure.runners.base import DependencyGraphRunner, evaluate_call, handle_result -# from quantdsl.domain.services.dependency_graphs import get_dependency_values -# from quantdsl.domain.model.call_specification import CallSpecification -# -# -# class SingleThreadedDependencyGraphRunner(DependencyGraphRunner): -# -# def run(self, **kwargs): -# super(SingleThreadedDependencyGraphRunner, self).run(**kwargs) -# self.call_queue = six.moves.queue.Queue() -# self.result_queue = six.moves.queue.Queue() -# self.calls_dict = self.dependency_graph.call_requirements.copy() -# self.requirements = self.dependency_graph.requirements.copy() -# self.dependents = self.dependency_graph.dependents.copy() -# # Put the leaves on the execution queue. -# for call_requirement_id in self.dependency_graph.leaf_ids: -# self.call_queue.put(call_requirement_id) -# # Loop over the required call queue. -# while not self.call_queue.empty(): -# # Get a waiting call requirement from the queue. -# call_requirement_id = self.call_queue.get() -# -# # Get the call attributes. -# dsl_source, effective_present_time = self.calls_dict[call_requirement_id] -# evaluation_kwds = self.get_evaluation_kwds(dsl_source, effective_present_time) -# dependency_values = get_dependency_values(call_requirement_id, self.requirements, self.results_repo) -# -# # Evaluate the call. -# call_spec = CallSpecification( -# id=call_requirement_id, -# dsl_expr_str=dsl_source, -# effective_present_time=effective_present_time, -# evaluation_kwds=evaluation_kwds, -# dependency_values=dependency_values -# ) -# -# evaluate_call(call_spec, self.result_queue) -# -# while not self.result_queue.empty(): -# call_requirement_id, result_value = self.result_queue.get() -# -# handle_result(call_requirement_id, result_value, self.results_repo, self.dependents, -# self.requirements, self.call_queue) -# -# self.call_count += 1 diff --git a/quantdsl/tests/test_application_with_greenthreads_and_pythonobjects.py b/quantdsl/tests/test_application_with_greenthreads_and_pythonobjects.py deleted file mode 100644 index 0825cc5..0000000 --- a/quantdsl/tests/test_application_with_greenthreads_and_pythonobjects.py +++ /dev/null @@ -1,14 +0,0 @@ -import gevent - -from quantdsl.application.with_greenthreads_and_python_objects import \ - QuantDslApplicationWithGreenThreadsAndPythonObjects -from quantdsl.tests.test_application import TestCase, ContractValuationTests - - -class TestApplicationWithGreenThreadsAndPythonObjects(TestCase, ContractValuationTests): - - def setup_application(self): - self.app = QuantDslApplicationWithGreenThreadsAndPythonObjects() - - def sleep(self, interval): - gevent.sleep(interval) diff --git a/quantdsl/tests/test_application_with_multiprocessing_and_cassandra.py b/quantdsl/tests/test_application_with_multiprocessing_and_cassandra.py deleted file mode 100644 index 83ca0af..0000000 --- a/quantdsl/tests/test_application_with_multiprocessing_and_cassandra.py +++ /dev/null @@ -1,37 +0,0 @@ -# import unittest -# -# from cassandra import OperationTimedOut -# from cassandra.cqlengine.management import drop_keyspace -# from eventsourcing.infrastructure.stored_events.cassandra_stored_events import create_cassandra_keyspace_and_tables -# -# from quantdsl.application.with_cassandra import DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE -# from quantdsl.application.with_multiprocessing_and_cassandra import QuantDslApplicationWithMultiprocessingAndCassandra -# from quantdsl.test_application import TestCase, ContractValuationTests -# -# -# class TestQuantDslApplicationWithMultiprocessingAndCassandra(TestCase, ContractValuationTests): -# -# def tearDown(self): -# # Drop keyspace before closing the application. -# drop_keyspace(DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE) -# super(TestQuantDslApplicationWithMultiprocessingAndCassandra, self).tearDown() -# -# def setup_application(self): -# self.app = QuantDslApplicationWithMultiprocessingAndCassandra(num_workers=self.NUMBER_WORKERS) -# try: -# # Create Cassandra keyspace and tables. -# create_cassandra_keyspace_and_tables(DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE) -# except OperationTimedOut: -# try: -# drop_keyspace(DEFAULT_QUANTDSL_CASSANDRA_KEYSPACE) -# except: -# pass -# self.app.close() -# raise -# -# -# -# if __name__ == '__main__': -# unittest.main() -# -# diff --git a/quantdsl/tests/test_application_with_multiprocessing_and_sqlalchemy.py b/quantdsl/tests/test_application_with_multiprocessing_and_sqlalchemy.py deleted file mode 100644 index 03eb63c..0000000 --- a/quantdsl/tests/test_application_with_multiprocessing_and_sqlalchemy.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -from tempfile import NamedTemporaryFile - -import os - -from quantdsl.application.with_multiprocessing_and_sqlalchemy import QuantDslApplicationWithMultiprocessingAndSQLAlchemy -from quantdsl.tests.test_application import TestCase, ContractValuationTests - - -class TestQuantDslApplicationWithMultiprocessingAndSQLAlchemy(TestCase, ContractValuationTests): - - def setup_application(self): - tempfile = NamedTemporaryFile() - self.tempfile_name = tempfile.name - tempfile.close() - self.app = QuantDslApplicationWithMultiprocessingAndSQLAlchemy( - num_workers=self.NUMBER_WORKERS, - db_uri='sqlite:///'+self.tempfile_name - ) - - def tearDown(self): - super(TestQuantDslApplicationWithMultiprocessingAndSQLAlchemy, self).tearDown() - os.unlink(self.tempfile_name) - - -if __name__ == '__main__': - unittest.main() - - diff --git a/quantdsl/tests/test_application_with_singlethread_and_sqlalchemy.py b/quantdsl/tests/test_application_with_singlethread_and_sqlalchemy.py deleted file mode 100644 index c19b615..0000000 --- a/quantdsl/tests/test_application_with_singlethread_and_sqlalchemy.py +++ /dev/null @@ -1,8 +0,0 @@ -from quantdsl.application.with_sqlalchemy import QuantDslApplicationWithSQLAlchemy -from quantdsl.tests.test_application import TestCase, ContractValuationTests - - -class TestQuantDslApplicationWithSQLAlchemy(TestCase, ContractValuationTests): - - def setup_application(self): - self.app = QuantDslApplicationWithSQLAlchemy(db_uri='sqlite:///:memory:') diff --git a/quantdsl/tests/test_distributed_dependency_graph_runner.py b/quantdsl/tests/test_distributed_dependency_graph_runner.py deleted file mode 100644 index 1d6d674..0000000 --- a/quantdsl/tests/test_distributed_dependency_graph_runner.py +++ /dev/null @@ -1,111 +0,0 @@ -# import datetime -# import os -# import sys -# import unittest -# from subprocess import Popen -# -# import scipy -# from pytz import utc -# -# from quantdsl.domain.model.dependency_graph import DependencyGraph -# from quantdsl.application.main import get_quantdsl_app -# from quantdsl.domain.model.call_result import CallResult -# from quantdsl.domain.model.simulated_price import register_simulated_price -# from quantdsl.domain.services.uuids import create_uuid4 -# from quantdsl.infrastructure.celery.tasks import celery_evaluate_call -# from quantdsl.infrastructure.runners.distributed import DistributedDependencyGraphRunner -# from quantdsl.services import dsl_compile -# -# -# class TestDistributedDependencyGraphRunner(unittest.TestCase): -# -# def test_evaluate_call(self): -# # Check the example task works directly. -# # - set up the call requirement -# app = get_quantdsl_app() -# call_id = create_uuid4() -# app.register_call_requirement(call_id, '1 + 2', datetime.datetime.now()) -# app.register_call_dependencies(call_id, []) -# app.register_call_dependents(call_id, []) -# -# celery_evaluate_call(call_id) -# call_result = app.call_result_repo[call_id] -# assert isinstance(call_result, CallResult) -# self.assertEqual(call_result.result_value, 3) -# -# # def _test_handle_result(self): -# # app = get_quantdsl_app() -# # call_id = create_uuid4() -# # app.register_call_requirement(call_id, '1 + 2', datetime.datetime.now()) -# # app.register_call_dependencies(call_id, []) -# # app.register_call_dependents(call_id, []) -# # celery_handle_result(call_id, 3) -# # return -# -# def test_distributed_dependency_graph_runner(self): -# # Setup the contract. -# # - branching function calls -# dsl_source = """ -# def Swing(starts, ends, underlying, quantity): -# if (quantity == 0) or (starts >= ends): -# 0 -# else: -# Wait(starts, Choice( -# Swing(starts + TimeDelta('1d'), ends, underlying, quantity - 1) + Fixing(starts, underlying), -# Swing(starts + TimeDelta('1d'), ends, underlying, quantity) -# )) -# Swing(Date('2011-01-01'), Date('2011-01-03'), 10, 50) -# """ -# -# # Generate the dependency graph. -# dependency_graph = dsl_compile(dsl_source, is_parallel=True) -# assert isinstance(dependency_graph, DependencyGraph) -# -# # Remember the number of stubbed exprs - will check it after the evaluation. -# actual_len_stubbed_exprs = len(dependency_graph.call_requirements) -# -# kwds = { -# 'interest_rate': 0, -# 'present_time': datetime.datetime(2011, 1, 1, tzinfo=utc), -# 'simulation_id': create_uuid4(), -# } -# app = get_quantdsl_app() -# market_simulation = app.register_market_simulation({}) -# -# market_names = ['#1'] -# for market_name in market_names: -# # NB Need enough days to cover the date range in the dsl_source. -# for i in range(0, 10): -# dt = datetime.datetime(2011, 1, 1, tzinfo=utc) + datetime.timedelta(1) * i -# value = scipy.array([10] * 2000) -# register_simulated_price(market_simulation.id, market_name, fixing_date=dt) -# -# # Check we've got a path to the 'celery' command line program (hopefully it's next to this python executable). -# celery_script_path = os.path.join(os.path.dirname(sys.executable), 'celery') -# self.assertTrue(os.path.exists(celery_script_path), celery_script_path) -# -# # Check the example task returns correct result (this assumes the celery worker IS running). -# # - invoke a celery worker process as a subprocess -# worker_cmd = [celery_script_path, 'worker', '-A', 'quantdsl.infrastructure.celery.tasks', '-l', 'info'] -# # - its usage as a context manager causes a wait for it to finish after it has been terminated -# with Popen(worker_cmd) as worker: -# try: -# # Evaluate the dependency graph. -# runner = DistributedDependencyGraphRunner(dependency_graph, app=app) -# dsl_value = runner.evaluate(**kwds) -# -# # Get the mean of the value, if it has one. -# if isinstance(dsl_value, scipy.ndarray): -# dsl_value = dsl_value.mean() -# -# # Check the value is expected. -# expected_value = 20 -# self.assertEqual(dsl_value, expected_value) -# -# # Check the number of stubbed exprs is expected. -# expected_len_stubbed_exprs = 7 -# self.assertEqual(actual_len_stubbed_exprs, expected_len_stubbed_exprs) -# -# finally: -# # Shutdown the celery worker. -# worker.terminate() From a26896edde53ee125a5c7a89a7aa7a9ead3fe121 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 14:10:03 +0100 Subject: [PATCH 02/52] Added quantdsl/lib to coverage omit list. --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 712f10a..83d9dba 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,4 @@ omit = */python?.?/* */test_?.py */tests/* + */quantdsl/lib/* From 1af67605d65c2ad2288627591618bbd3674655fd Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 14:20:30 +0100 Subject: [PATCH 03/52] Improved test, to cover defining required type with a sequence. --- quantdsl/tests/test_semantics.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index afdf37d..ebde2e3 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -38,6 +38,10 @@ def test_assert_args_arg(self): with self.assertRaises(DslSyntaxError): self.obj.assert_args_arg([[1, 'a']], 0, [int]) + self.obj.assert_args_arg([1], 0, (int, float)) + with self.assertRaises(DslSyntaxError): + self.obj.assert_args_arg(['1'], 0, (int, float)) + def test_pprint(self): text = self.obj.pprint() self.assertEqual(text, "Subclass()") From 11a77bc0c04fc2f9f948b134f4021d1328def6a0 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 14:25:21 +0100 Subject: [PATCH 04/52] Fixed indent. --- quantdsl/tests/test_semantics.py | 170 +++++++++++++++---------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index ebde2e3..07f25bf 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -51,88 +51,88 @@ def test_pprint(self): 1 )""") - class TestString(TestCase): - def test(self): - obj = String('a') - self.assertEqual(obj.value, 'a') - - obj = String('b') - self.assertEqual(obj.value, 'b') - - with self.assertRaises(DslSyntaxError): - String() - with self.assertRaises(DslSyntaxError): - String('a', 'b') - with self.assertRaises(DslSyntaxError): - String(1) - - class TestNumber(TestCase): - def test(self): - # Integers are ok. - obj = Number(1) - self.assertEqual(obj.value, 1) - - # Floats are ok. - obj = Number(1.1) - self.assertEqual(obj.value, 1.1) - - # Numpy arrays are ok. - obj = Number(array([1, 2])) - self.assertEqual(list(obj.value), list(array([1, 2]))) - - # No args is not ok. - with self.assertRaises(DslSyntaxError): - Number() - - # Two args is not ok. - with self.assertRaises(DslSyntaxError): - Number(1, 1.1) - - # A list is not ok. - with self.assertRaises(DslSyntaxError): - Number([1, 1.1]) - - # A string is not ok. - with self.assertRaises(DslSyntaxError): - Number('1') - - class TestDate(TestCase): - def test(self): - # A Python string is ok. - obj = Date('2011-1-1') - self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) - - # A Quant DSL String is ok. - obj = Date(String('2011-1-1')) - self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) - - # A date is ok. - obj = Date(datetime.date(2011, 1, 1)) - self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) - - # A datetime is ok. - obj = Date(datetime.datetime(2011, 1, 1)) - self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) - - # No args is not ok. - with self.assertRaises(DslSyntaxError): - Date() - - # Two args is not ok. - with self.assertRaises(DslSyntaxError): - Date(1, 1.1) - - # A string that doesn't look like a date is not ok. - with self.assertRaises(DslSyntaxError): - Date('1') - - class TestTimeDelta(TestCase): - def test(self): - obj = TimeDelta(String('1d')) - self.assertEqual(obj.value, relativedelta(days=1)) - obj = TimeDelta(String('2d')) - self.assertEqual(obj.value, relativedelta(days=2)) - obj = TimeDelta(String('1m')) - self.assertEqual(obj.value, relativedelta(months=1)) - obj = TimeDelta(String('1y')) - self.assertEqual(obj.value, relativedelta(years=1)) +class TestString(TestCase): + def test(self): + obj = String('a') + self.assertEqual(obj.value, 'a') + + obj = String('b') + self.assertEqual(obj.value, 'b') + + with self.assertRaises(DslSyntaxError): + String() + with self.assertRaises(DslSyntaxError): + String('a', 'b') + with self.assertRaises(DslSyntaxError): + String(1) + +class TestNumber(TestCase): + def test(self): + # Integers are ok. + obj = Number(1) + self.assertEqual(obj.value, 1) + + # Floats are ok. + obj = Number(1.1) + self.assertEqual(obj.value, 1.1) + + # Numpy arrays are ok. + obj = Number(array([1, 2])) + self.assertEqual(list(obj.value), list(array([1, 2]))) + + # No args is not ok. + with self.assertRaises(DslSyntaxError): + Number() + + # Two args is not ok. + with self.assertRaises(DslSyntaxError): + Number(1, 1.1) + + # A list is not ok. + with self.assertRaises(DslSyntaxError): + Number([1, 1.1]) + + # A string is not ok. + with self.assertRaises(DslSyntaxError): + Number('1') + +class TestDate(TestCase): + def test(self): + # A Python string is ok. + obj = Date('2011-1-1') + self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) + + # A Quant DSL String is ok. + obj = Date(String('2011-1-1')) + self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) + + # A date is ok. + obj = Date(datetime.date(2011, 1, 1)) + self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) + + # A datetime is ok. + obj = Date(datetime.datetime(2011, 1, 1)) + self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) + + # No args is not ok. + with self.assertRaises(DslSyntaxError): + Date() + + # Two args is not ok. + with self.assertRaises(DslSyntaxError): + Date(1, 1.1) + + # A string that doesn't look like a date is not ok. + with self.assertRaises(DslSyntaxError): + Date('1-20').value + +class TestTimeDelta(TestCase): + def test(self): + obj = TimeDelta(String('1d')) + self.assertEqual(obj.value, relativedelta(days=1)) + obj = TimeDelta(String('2d')) + self.assertEqual(obj.value, relativedelta(days=2)) + obj = TimeDelta(String('1m')) + self.assertEqual(obj.value, relativedelta(months=1)) + obj = TimeDelta(String('1y')) + self.assertEqual(obj.value, relativedelta(years=1)) From ec5ca78d205ae2a70defed257dbf8b5c20cf60db Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 15:09:58 +0100 Subject: [PATCH 05/52] Improved test coverage for TimeDelta semantic class. Added test for str() of DSL objects. --- quantdsl/semantics.py | 6 ++-- quantdsl/tests/test_semantics.py | 52 +++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index a6a60c4..7bc8090 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -243,16 +243,16 @@ class TimeDelta(DslConstant): required_type = (String, datetime.timedelta, relativedelta) def __str__(self, indent=0): - return "{}('{}')".format(self.__class__.__name__, self.value) + return "{}({})".format(self.__class__.__name__, self._args[0]) def parse(self, value, regex=re.compile(r'((?P\d+?)d|(?P\d+?)m|(?P\d+?)y)?')): if isinstance(value, String): duration_str = value.evaluate() parts = regex.match(duration_str) - if not parts: - raise DslSyntaxError('invalid time delta string', duration_str, node=self.node) parts = parts.groupdict() params = dict((name, int(param)) for (name, param) in six.iteritems(parts) if param) + if not params: + raise DslSyntaxError('invalid "time delta" string', duration_str, node=self.node) return relativedelta(**params) elif isinstance(value, datetime.timedelta): return value diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index 07f25bf..3a5f76e 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -5,7 +5,7 @@ from scipy import array from quantdsl.exceptions import DslSyntaxError -from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta +from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name class Subclass(DslObject): @@ -42,17 +42,13 @@ def test_assert_args_arg(self): with self.assertRaises(DslSyntaxError): self.obj.assert_args_arg(['1'], 0, (int, float)) - def test_pprint(self): - text = self.obj.pprint() - self.assertEqual(text, "Subclass()") - self.assertEqual(Subclass(Subclass()).pprint(), "Subclass(Subclass())") - self.assertEqual(Subclass(Subclass(), 1).pprint(), """Subclass( - Subclass(), - 1 -)""") + def test_str(self): + self.assertEqual(str(self.obj), "Subclass()") + self.assertEqual(str(Subclass(Subclass())), "Subclass(Subclass())") + class TestString(TestCase): - def test(self): + def test_value(self): obj = String('a') self.assertEqual(obj.value, 'a') @@ -66,8 +62,14 @@ def test(self): with self.assertRaises(DslSyntaxError): String(1) + def test_str(self): + obj = String('a') + self.assertEqual(str(obj), "'a'") + self.assertEqual(str(Subclass(obj)), "Subclass('a')") + + class TestNumber(TestCase): - def test(self): + def test_value(self): # Integers are ok. obj = Number(1) self.assertEqual(obj.value, 1) @@ -96,8 +98,14 @@ def test(self): with self.assertRaises(DslSyntaxError): Number('1') + def test_str(self): + obj = Number(1) + self.assertEqual(str(obj), '1') + self.assertEqual(str(Subclass(obj)), 'Subclass(1)') + + class TestDate(TestCase): - def test(self): + def test_value(self): # A Python string is ok. obj = Date('2011-1-1') self.assertEqual(obj.value, datetime.datetime(2011, 1, 1)) @@ -124,10 +132,17 @@ def test(self): # A string that doesn't look like a date is not ok. with self.assertRaises(DslSyntaxError): - Date('1-20').value + Date('1') + + def test_str(self): + obj = Date(String('2011-1-1')) + self.assertEqual(str(obj), "Date('2011-01-01')") + self.assertEqual(str(Subclass(obj)), "Subclass(Date('2011-01-01'))") + class TestTimeDelta(TestCase): - def test(self): + def test_value(self): + # Days, months, or years is ok. obj = TimeDelta(String('1d')) self.assertEqual(obj.value, relativedelta(days=1)) obj = TimeDelta(String('2d')) @@ -136,3 +151,12 @@ def test(self): self.assertEqual(obj.value, relativedelta(months=1)) obj = TimeDelta(String('1y')) self.assertEqual(obj.value, relativedelta(years=1)) + + # An invalid time delta string is not ok. + with self.assertRaises(DslSyntaxError): + TimeDelta(String('1j')) + + def test_str(self): + obj = TimeDelta(String('1d')) + self.assertEqual(str(obj), "TimeDelta('1d')") + self.assertEqual(str(Subclass(obj)), "Subclass(TimeDelta('1d'))") From ad176c2454491b5a7c60d78d82ef6864dce8c27e Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 15:10:33 +0100 Subject: [PATCH 06/52] Removed unused stuff from 'install_requires' in setup.py. --- setup.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/setup.py b/setup.py index 2fdbc2b..d7931cb 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,6 @@ zip_safe=False, install_requires=[ 'argh', - 'celery', 'mock==1.0.1', 'scipy', 'python-dateutil==2.2', @@ -33,15 +32,6 @@ 'pytz', 'blist', ], - extras_require={ - 'test': [ - 'sqlalchemy', - 'gevent', - ], - 'sqlalchemy': [ - 'sqlalchemy', - ], - }, scripts=[], author='John Bywater', From 6e2b49864f0c8eefffafaae85f16ceee20bae877 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 15:17:50 +0100 Subject: [PATCH 07/52] Removed gevent stuff. --- quantdsl/domain/services/contract_valuations.py | 7 ------- requirements.txt | 3 --- setup.py | 1 + 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/quantdsl/domain/services/contract_valuations.py b/quantdsl/domain/services/contract_valuations.py index 12a12d3..c37926a 100644 --- a/quantdsl/domain/services/contract_valuations.py +++ b/quantdsl/domain/services/contract_valuations.py @@ -1,8 +1,6 @@ from multiprocessing.pool import Pool -import gevent from eventsourcing.domain.model.events import publish -from gevent.queue import Queue from quantdsl.domain.model.call_dependencies import CallDependencies from quantdsl.domain.model.call_dependents import CallDependents @@ -154,7 +152,6 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo for call_id in call_leafs.leaf_ids: call_evaluation_queue.put((contract_specification_id, contract_valuation_id, call_id)) - gevent.sleep(0) def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, call_requirement_repo, @@ -163,8 +160,6 @@ def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, cal call_result_lock, compute_pool=None, result_counters=None, usage_counters=None): while True: item = call_evaluation_queue.get() - if isinstance(call_evaluation_queue, gevent.queue.Queue): - gevent.sleep(0) try: contract_specification_id, contract_valuation_id, call_id = item @@ -286,7 +281,6 @@ def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specifica next_call_ids = [] for next_call_id in ready_generator: call_evaluation_queue.put((contract_specification_id, contract_valuation_id, next_call_id)) - gevent.sleep(0) finally: # Unlock the results. @@ -450,7 +444,6 @@ def compute_call_result(contract_valuation, call_requirement, market_simulation, present_time, simulated_value_dict, perturbation_dependencies.dependencies, dependency_results, market_simulation.path_count, market_simulation.perturbation_factor), ) - gevent.sleep(0.0001) result_value, perturbed_values = async_result.get() # Return the result. diff --git a/requirements.txt b/requirements.txt index 2fc4078..01e3019 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,11 @@ argh -celery mock==1.0.1 scipy python-dateutil==2.2 requests six==1.7.3 eventsourcing==0.9.4 -sqlalchemy filelock -gevent==1.1rc3 pytz blist importlib diff --git a/setup.py b/setup.py index d7931cb..4147b37 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ 'eventsourcing==0.9.4', 'pytz', 'blist', + 'importlib', ], scripts=[], From be0f377773f06c7f8f43458d26f2589fc44bca07 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 15:34:38 +0100 Subject: [PATCH 08/52] Refactored pprint() and __str__() methods. --- quantdsl/semantics.py | 67 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index 7bc8090..19016d2 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -32,29 +32,13 @@ def __init__(self, *args, **kwds): self.node = kwds.pop('node', None) self.validate(args) self._args = list(args) + self._hash = None def __str__(self, indent=0): """ Returns DSL source code, that can be parsed to generate a clone of self. """ - return "%s(%s)" % (self.__class__.__name__, ", ".join([str(i) for i in self._args])) - - @property - def hash(self): - """ - Creates a hash that is unique for this fragment of DSL. - """ - if not hasattr(self, '_hash'): - hashes = "" - for arg in self._args: - if isinstance(arg, list): - arg = tuple(arg) - hashes += str(hash(arg)) - self._hash = hash(hashes) - return self._hash - - def __hash__(self): - return self.hash + return self.pprint(' ') def pprint(self, indent=''): msg = self.__class__.__name__ + "(" @@ -80,6 +64,23 @@ def pprint(self, indent=''): msg += ")" return msg + @property + def hash(self): + """ + Creates a hash that is unique for this fragment of DSL. + """ + if self._hash is None: + hashes = "" + for arg in self._args: + if isinstance(arg, list): + arg = tuple(arg) + hashes += str(hash(arg)) + self._hash = hash(hashes) + return self._hash + + def __hash__(self): + return self.hash + @abstractmethod def validate(self, args): """ @@ -184,7 +185,7 @@ def discount(self, value, date, **kwds): class DslConstant(DslExpression): required_type = None - def __str__(self, indent=0): + def pprint(self, indent=0): return repr(self.value) def validate(self, args): @@ -217,7 +218,7 @@ class Number(DslConstant): class Date(DslConstant): required_type = six.string_types + (String, datetime.date, datetime.datetime) - def __str__(self, indent=0): + def pprint(self, indent=0): return "Date('%04d-%02d-%02d')" % (self.value.year, self.value.month, self.value.day) def parse(self, value): @@ -242,7 +243,7 @@ def parse(self, value): class TimeDelta(DslConstant): required_type = (String, datetime.timedelta, relativedelta) - def __str__(self, indent=0): + def pprint(self, indent=0): return "{}({})".format(self.__class__.__name__, self._args[0]) def parse(self, value, regex=re.compile(r'((?P\d+?)d|(?P\d+?)m|(?P\d+?)y)?')): @@ -265,7 +266,7 @@ def parse(self, value, regex=re.compile(r'((?P\d+?)d|(?P\d+?)m|(?P class UnaryOp(DslExpression): opchar = None - def __str__(self, indent=0): + def pprint(self, indent=0): return str(self.opchar) + str(self.operand) def validate(self, args): @@ -338,7 +339,7 @@ def op(self, left, right): Returns result of operating on two args. """ - def __str__(self, indent=0): + def pprint(self, indent=0): if self.opchar: def makeStr(dsl_expr): dslString = str(dsl_expr) @@ -476,7 +477,7 @@ def op(self, left, right): class Name(DslExpression): - def __str__(self, indent=0): + def pprint(self, indent=0): return self.name def validate(self, args): @@ -536,7 +537,7 @@ class Stub(Name): with the value of another expression in a dependency graph. """ - def __str__(self, indent=0): + def pprint(self, indent=0): # Can't just return a Python string, like with Names, because this # is normally a UUID, and UUIDs are not valid Python variable names # because they have dashes and sometimes start with numbers. @@ -562,7 +563,7 @@ class FunctionDef(DslObject): are assignments. """ - def __str__(self, indent=0): + def pprint(self, indent=0): indent_spaces = 4 * ' ' msg = "" for decorator_name in self.decorator_names: @@ -570,7 +571,7 @@ def __str__(self, indent=0): msg += "def %s(%s):\n" % (self.name, ", ".join(self.call_arg_names)) if isinstance(self.body, DslObject): try: - msg += indent_spaces + self.body.__str__(indent=indent + 1) + msg += indent_spaces + self.body.pprint(indent=indent + 1) except TypeError: raise DslSystemError("DSL object can't handle indent: %s" % type(self.body)) else: @@ -722,7 +723,7 @@ def create_hash(self, obj): class FunctionCall(DslExpression): - def __str__(self, indent=0): + def pprint(self, indent=0): return "%s(%s)" % (self.functionDefName, ", ".join([str(arg) for arg in self.callArgExprs])) @@ -851,7 +852,7 @@ def evaluate(self, **kwds): class If(BaseIf): - def __str__(self, indent=0): + def pprint(self, indent=0): indentation = indent * 4 * ' ' msg = "\n" @@ -880,7 +881,7 @@ class IfExp(If): Special case of If, where if-else clause is one expression (no elif support). """ - def __str__(self, indent=0): + def pprint(self, indent=0): return "%s if %s else %s" % (self.body, self.test, self.orelse) @@ -903,7 +904,7 @@ class Compare(DslExpression): 'GtE': '>=', } - def __str__(self, indent=0): + def pprint(self, indent=0): return str(self.left) + ' ' \ + " ".join( [str(self.opcodes[op]) + ' ' + str(right) for (op, right) in zip(self.op_names, self.comparators)]) @@ -951,7 +952,7 @@ class Module(DslObject): def __init__(self, *args, **kwds): super(Module, self).__init__(*args, **kwds) - def __str__(self, indent=0): + def pprint(self, indent=0): return "\n".join([str(statement) for statement in self.body]) def validate(self, args): @@ -1247,7 +1248,7 @@ class Fixing(StochasticObject, DatedDslObject, DslExpression): A fixing defines the 'present_time' used for evaluating its expression. """ - def __str__(self, indent=0): + def pprint(self, indent=0): return "%s('%04d-%02d-%02d', %s)" % ( self.__class__.__name__, self.date.year, From 07b526928c01167e60c233570c555b5746a60c1e Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 15:50:02 +0100 Subject: [PATCH 09/52] Fixed whitespace. --- quantdsl/tests/test_least_squares.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quantdsl/tests/test_least_squares.py b/quantdsl/tests/test_least_squares.py index 5d06d01..0869092 100644 --- a/quantdsl/tests/test_least_squares.py +++ b/quantdsl/tests/test_least_squares.py @@ -33,4 +33,4 @@ def test_fit2(self): fixture_x=[[0, 1, 2], [3, 4, 5]], fixture_y=[0, 1, 2], expected_values=[0, 1, 2], - ) \ No newline at end of file + ) From c258b2a0b3df5e1939992c4016dc782366941548 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Thu, 14 Sep 2017 15:50:36 +0100 Subject: [PATCH 10/52] Fixed indent arg implementation in pprint() methods. --- quantdsl/semantics.py | 89 +++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index 19016d2..dff221f 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -34,11 +34,11 @@ def __init__(self, *args, **kwds): self._args = list(args) self._hash = None - def __str__(self, indent=0): + def __str__(self): """ Returns DSL source code, that can be parsed to generate a clone of self. """ - return self.pprint(' ') + return self.pprint() def pprint(self, indent=''): msg = self.__class__.__name__ + "(" @@ -139,11 +139,11 @@ def find_instances(self, dsl_type): if isinstance(arg, DslObject): for dsl_obj in arg.find_instances(dsl_type): yield dsl_obj - # elif isinstance(arg, list): - # for arg in arg: - # if isinstance(arg, DslObject): - # for dsl_obj in arg.list_instances(dsl_type): - # yield dsl_obj + # elif isinstance(arg, list): + # for arg in arg: + # if isinstance(arg, DslObject): + # for dsl_obj in arg.list_instances(dsl_type): + # yield dsl_obj def reduce(self, dsl_locals, dsl_globals, effective_present_time=None, pending_call_stack=None): """ @@ -185,7 +185,7 @@ def discount(self, value, date, **kwds): class DslConstant(DslExpression): required_type = None - def pprint(self, indent=0): + def pprint(self, indent=''): return repr(self.value) def validate(self, args): @@ -218,7 +218,7 @@ class Number(DslConstant): class Date(DslConstant): required_type = six.string_types + (String, datetime.date, datetime.datetime) - def pprint(self, indent=0): + def pprint(self, indent=''): return "Date('%04d-%02d-%02d')" % (self.value.year, self.value.month, self.value.day) def parse(self, value): @@ -243,7 +243,7 @@ def parse(self, value): class TimeDelta(DslConstant): required_type = (String, datetime.timedelta, relativedelta) - def pprint(self, indent=0): + def pprint(self, indent=''): return "{}({})".format(self.__class__.__name__, self._args[0]) def parse(self, value, regex=re.compile(r'((?P\d+?)d|(?P\d+?)m|(?P\d+?)y)?')): @@ -266,7 +266,7 @@ def parse(self, value, regex=re.compile(r'((?P\d+?)d|(?P\d+?)m|(?P class UnaryOp(DslExpression): opchar = None - def pprint(self, indent=0): + def pprint(self, indent=''): return str(self.opchar) + str(self.operand) def validate(self, args): @@ -339,7 +339,7 @@ def op(self, left, right): Returns result of operating on two args. """ - def pprint(self, indent=0): + def pprint(self, indent=''): if self.opchar: def makeStr(dsl_expr): dslString = str(dsl_expr) @@ -347,9 +347,10 @@ def makeStr(dsl_expr): dslString = "(" + dslString + ")" return dslString - return makeStr(self.left) + " " + self.opchar + " " + makeStr(self.right) + text = makeStr(self.left) + " " + self.opchar + " " + makeStr(self.right) else: - return '%s(%s, %s)' % (self.__class__.__name__, self.left, self.right) + text = '%s(%s, %s)' % (self.__class__.__name__, self.left, self.right) + return indent + text def validate(self, args): self.assert_args_len(args, required_len=2) @@ -477,7 +478,7 @@ def op(self, left, right): class Name(DslExpression): - def pprint(self, indent=0): + def pprint(self, indent=''): return self.name def validate(self, args): @@ -537,7 +538,7 @@ class Stub(Name): with the value of another expression in a dependency graph. """ - def pprint(self, indent=0): + def pprint(self, indent=''): # Can't just return a Python string, like with Names, because this # is normally a UUID, and UUIDs are not valid Python variable names # because they have dashes and sometimes start with numbers. @@ -563,15 +564,14 @@ class FunctionDef(DslObject): are assignments. """ - def pprint(self, indent=0): - indent_spaces = 4 * ' ' + def pprint(self, indent=''): msg = "" for decorator_name in self.decorator_names: msg += "@" + decorator_name + "\n" msg += "def %s(%s):\n" % (self.name, ", ".join(self.call_arg_names)) if isinstance(self.body, DslObject): try: - msg += indent_spaces + self.body.pprint(indent=indent + 1) + msg += self.body.pprint(indent=indent + ' ') except TypeError: raise DslSystemError("DSL object can't handle indent: %s" % type(self.body)) else: @@ -709,7 +709,7 @@ def create_hash(self, obj): if isinstance(obj, relativedelta): return hash(repr(obj)) if isinstance(obj, ( - int, float, six.string_types, datetime.datetime, datetime.date, datetime.timedelta, relativedelta)): + int, float, six.string_types, datetime.datetime, datetime.date, datetime.timedelta, relativedelta)): return hash(obj) if isinstance(obj, dict): return hash(tuple(sorted([(a, self.create_hash(b)) for a, b in obj.items()]))) @@ -723,9 +723,9 @@ def create_hash(self, obj): class FunctionCall(DslExpression): - def pprint(self, indent=0): - return "%s(%s)" % (self.functionDefName, - ", ".join([str(arg) for arg in self.callArgExprs])) + def pprint(self, indent=''): + return indent + "%s(%s)" % (self.functionDefName, + ", ".join([str(arg) for arg in self.callArgExprs])) def validate(self, args): self.assert_args_len(args, required_len=2) @@ -852,27 +852,24 @@ def evaluate(self, **kwds): class If(BaseIf): - def pprint(self, indent=0): - indentation = indent * 4 * ' ' - + def pprint(self, indent=''): msg = "\n" - msg += indentation + "if %s:\n" % self.test - msg += indentation + " %s\n" % self.body - - msg += self.orelse_to_str(self.orelse, indentation) + msg += indent + "if %s:\n" % self.test + msg += indent + " %s\n" % self.body + msg += self.orelse_to_str(self.orelse, indent) return msg - def orelse_to_str(self, orelse, indentation): + def orelse_to_str(self, orelse, indent): msg = '' if isinstance(orelse, If): - msg += indentation + "elif %s:\n" % orelse.test - msg += indentation + " %s\n" % orelse.body + msg += indent + "elif %s:\n" % orelse.test + msg += indent + " %s\n" % orelse.body # Recurse down "linked list" of alternatives... - msg += self.orelse_to_str(orelse.orelse, indentation) + msg += self.orelse_to_str(orelse.orelse, indent) else: # ...until we reach the final alternative. - msg += indentation + "else:\n" - msg += indentation + " %s\n" % orelse + msg += indent + "else:\n" + msg += indent + " %s\n" % orelse return msg @@ -881,8 +878,8 @@ class IfExp(If): Special case of If, where if-else clause is one expression (no elif support). """ - def pprint(self, indent=0): - return "%s if %s else %s" % (self.body, self.test, self.orelse) + def pprint(self, indent=''): + return indent + "%s if %s else %s" % (self.body, self.test, self.orelse) class Compare(DslExpression): @@ -904,10 +901,10 @@ class Compare(DslExpression): 'GtE': '>=', } - def pprint(self, indent=0): - return str(self.left) + ' ' \ - + " ".join( - [str(self.opcodes[op]) + ' ' + str(right) for (op, right) in zip(self.op_names, self.comparators)]) + def pprint(self, indent=''): + return indent + str(self.left) + ' ' + " ".join( + [str(self.opcodes[op]) + ' ' + str(right) for (op, right) in zip(self.op_names, self.comparators)] + ) def validate(self, args): self.assert_args_len(args, 3) @@ -952,8 +949,8 @@ class Module(DslObject): def __init__(self, *args, **kwds): super(Module, self).__init__(*args, **kwds) - def pprint(self, indent=0): - return "\n".join([str(statement) for statement in self.body]) + def pprint(self, indent=''): + return indent + "\n".join([str(statement) for statement in self.body]) def validate(self, args): self.assert_args_len(args, 2) @@ -1248,8 +1245,8 @@ class Fixing(StochasticObject, DatedDslObject, DslExpression): A fixing defines the 'present_time' used for evaluating its expression. """ - def pprint(self, indent=0): - return "%s('%04d-%02d-%02d', %s)" % ( + def pprint(self, indent=''): + return indent + "%s('%04d-%02d-%02d', %s)" % ( self.__class__.__name__, self.date.year, self.date.month, From 2e17e82a2e7ecea90bfdf8cde63eb955f353f1b4 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Fri, 15 Sep 2017 18:51:34 +0100 Subject: [PATCH 11/52] Added test for class DslError. --- quantdsl/semantics.py | 6 ++++ quantdsl/tests/test_exceptions.py | 22 +++++++++++++ quantdsl/tests/test_semantics.py | 51 ++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 quantdsl/tests/test_exceptions.py diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index dff221f..65778e3 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -319,6 +319,12 @@ def op(self, value): Returns value, or not value, according to implementation. """ + def pprint(self, indent=''): + operator = self.__class__.__name__.lower() + padded = ' ' + operator + ' ' + text = padded.join([str(i) for i in self._args[0]]) + return indent + '(' + text + ')' + class Or(BoolOp): def op(self, value): diff --git a/quantdsl/tests/test_exceptions.py b/quantdsl/tests/test_exceptions.py new file mode 100644 index 0000000..45e109f --- /dev/null +++ b/quantdsl/tests/test_exceptions.py @@ -0,0 +1,22 @@ +from unittest.case import TestCase + +from mock import Mock + +from quantdsl.exceptions import DslError + + +class TestDslError(TestCase): + def test(self): + error = DslError(error='error', descr='descr') + self.assertEqual(error.error, 'error') + self.assertEqual(error.descr, 'descr') + self.assertEqual(error.node, None) + self.assertEqual(error.lineno, None) + self.assertEqual(repr(error), 'error: descr') + + node = Mock() + node.lineno = 123 + error = DslError(error='error', descr='descr', node=node) + self.assertEqual(repr(error), 'error: descr (line 123)') + self.assertEqual(error.node, node) + self.assertEqual(error.lineno, 123) diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index 3a5f76e..d25a62a 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -5,7 +5,7 @@ from scipy import array from quantdsl.exceptions import DslSyntaxError -from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name +from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name, And, Or class Subclass(DslObject): @@ -160,3 +160,52 @@ def test_str(self): obj = TimeDelta(String('1d')) self.assertEqual(str(obj), "TimeDelta('1d')") self.assertEqual(str(Subclass(obj)), "Subclass(TimeDelta('1d'))") + + +class TestAnd(TestCase): + + def test_evaluate(self): + obj = And([Number(1), Number(1)]) + self.assertTrue(obj.evaluate()) + + obj = And([Number(1), Number(0)]) + self.assertFalse(obj.evaluate()) + + obj = And([Number(0), Number(1)]) + self.assertFalse(obj.evaluate()) + + obj = And([Number(0), Number(0)]) + self.assertFalse(obj.evaluate()) + + def test_str(self): + obj = And([Number(1), Number(2), Number(3)]) + self.assertEqual(str(obj), '(1 and 2 and 3)') + + +class TestOr(TestCase): + + def test_evaluate(self): + obj = Or([Number(1), Number(1)]) + self.assertTrue(obj.evaluate()) + + obj = Or([Number(1), Number(0)]) + self.assertTrue(obj.evaluate()) + + obj = Or([Number(0), Number(1)]) + self.assertTrue(obj.evaluate()) + + obj = Or([Number(0), Number(0)]) + self.assertFalse(obj.evaluate()) + + def test_str(self): + obj = Or([Number(1), Number(2), Number(3)]) + self.assertEqual(str(obj), '(1 or 2 or 3)') + + +class TestAndOr(TestCase): + + def test_str(self): + obj = And([Number(1), Or([Number(2), Number(3)])]) + self.assertEqual(str(obj), '(1 and (2 or 3))') + # Check the indentation isn't propagated. + self.assertEqual(obj.pprint(' '), ' (1 and (2 or 3))') From 311d67a3714d04aaaf7c1f3449a8dc4f98b292c3 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Fri, 15 Sep 2017 19:13:00 +0100 Subject: [PATCH 12/52] Added test for Add, Sub, Mult, and Div. --- quantdsl/semantics.py | 8 ---- quantdsl/tests/test_semantics.py | 66 +++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index 65778e3..15d8fea 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -374,14 +374,6 @@ def right(self): def evaluate(self, **kwds): left = self.left.evaluate(**kwds) right = self.right.evaluate(**kwds) - if isinstance(left, datetime.timedelta) and isinstance(right, float): - rightOrig = right - right = int(right) - assert rightOrig == right, "Can't %s timedelta and fractional number '%s'" % rightOrig - elif isinstance(right, datetime.timedelta) and isinstance(left, float): - leftOrig = left - left = int(left) - assert leftOrig == left, "Can't %s timedelta and fractional number '%s'" % leftOrig try: return self.op(left, right) except TypeError as e: diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index d25a62a..d3e6fcf 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -5,7 +5,7 @@ from scipy import array from quantdsl.exceptions import DslSyntaxError -from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name, And, Or +from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name, And, Or, Add, Sub, Mult, Div class Subclass(DslObject): @@ -209,3 +209,67 @@ def test_str(self): self.assertEqual(str(obj), '(1 and (2 or 3))') # Check the indentation isn't propagated. self.assertEqual(obj.pprint(' '), ' (1 and (2 or 3))') + + +class TestAdd(TestCase): + + def test_evaluate(self): + obj = Add(Number(1), Number(1)) + self.assertEqual(obj.evaluate(), 2) + + obj = Add(String('a'), String('b')) + self.assertEqual(obj.evaluate(), 'ab') + + obj = Add(Number(1), String('a')) + with self.assertRaises(DslSyntaxError): + obj.evaluate() + + +class TestSub(TestCase): + + def test_evaluate(self): + obj = Sub(Number(1), Number(1)) + self.assertEqual(obj.evaluate(), 0) + + obj = Sub(Number(1), String('a')) + with self.assertRaises(DslSyntaxError): + obj.evaluate() + + +class TestMul(TestCase): + + def test_evaluate(self): + obj = Mult(Number(2), Number(2)) + self.assertEqual(obj.evaluate(), 4) + + obj = Mult(Number(2), String('a')) + self.assertEqual(obj.evaluate(), 'aa') + + obj = Mult(Number(2.0), String('a')) + with self.assertRaises(DslSyntaxError): + obj.evaluate() + + obj = Mult(String('a'), String('a')) + with self.assertRaises(DslSyntaxError): + obj.evaluate() + + obj = Mult(Number(2.1), String('a')) + with self.assertRaises(DslSyntaxError): + obj.evaluate() + + +class TestDiv(TestCase): + def test_evaluate(self): + obj = Div(Number(5), Number(2)) + self.assertEqual(obj.evaluate(), 2.5) + + obj = Div(TimeDelta(String('2d')), Number(2)) + self.assertEqual(obj.evaluate(), relativedelta(days=1)) + + obj = Div(Number(5), Number(0)) + with self.assertRaises(ZeroDivisionError): + obj.evaluate() + + obj = Div(Number(2.1), String('a')) + with self.assertRaises(DslSyntaxError): + obj.evaluate() From 414f78e1c040c1239d35cd4ee251c1421971ef8a Mon Sep 17 00:00:00 2001 From: John Bywater Date: Fri, 15 Sep 2017 19:39:50 +0100 Subject: [PATCH 13/52] Added test cases for Min, Max, and Name. --- quantdsl/semantics.py | 67 +++++++++++++++++++------------- quantdsl/tests/test_semantics.py | 53 ++++++++++++++++++++++++- 2 files changed, 91 insertions(+), 29 deletions(-) diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index 15d8fea..0a0e636 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -206,6 +206,12 @@ def evaluate(self, **_): def parse(self, value): return value + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return not self.__eq__(other) + class String(DslConstant): required_type = six.string_types @@ -409,6 +415,30 @@ def op(self, left, right): return left / right +# Todo: Pow, Mod, FloorDiv don't have proofs, so shouldn't really be used for combining random variables? Either +# prevent usage with ndarray inputs, or do the proofs. :-) + +class Pow(BinOp): + opchar = '**' + + def op(self, left, right): + return left ** right + + +class Mod(BinOp): + opchar = '%' + + def op(self, left, right): + return left % right + + +class FloorDiv(BinOp): + opchar = '//' + + def op(self, left, right): + return left // right + + class NonInfixedBinOp(BinOp): def op(self, a, b): # Assume a and b have EITHER type ndarray, OR type int or float. @@ -423,9 +453,8 @@ def op(self, a, b): return self.scalar_op(a, b) elif (not aIsaNumber) and (not bIsaNumber): # Both are vectors. - if len(a) != len(b): - descr = "%s and %s" % (len(a), len(b)) - raise DslSystemError('Vectors have different length: ', descr, self.node) + msg = "Vectors have different length: %s and %s" % (len(a), len(b)) + assert len(a) == len(b), msg elif aIsaNumber and (not bIsaNumber): # Todo: Optimise with scipy.zeros() when a equals zero? a = scipy.array([a] * len(b)) @@ -434,6 +463,14 @@ def op(self, a, b): b = scipy.array([b] * len(a)) return self.vector_op(a, b) + @abstractmethod + def vector_op(self, a, b): + """Computes result of operation on vector values.""" + + @abstractmethod + def scalar_op(self, a, b): + """Computes result of operation on scalar values.""" + class Min(NonInfixedBinOp): def vector_op(self, a, b): @@ -451,30 +488,6 @@ def scalar_op(self, a, b): return max(a, b) -# Todo: Pow, Mod, FloorDiv don't have proofs, so shouldn't really be used for combining random variables? Either -# prevent usage with ndarray inputs, or do the proofs. :-) - -class Pow(BinOp): - opchar = '**' - - def op(self, left, right): - return left ** right - - -class Mod(BinOp): - opchar = '%' - - def op(self, left, right): - return left % right - - -class FloorDiv(BinOp): - opchar = '//' - - def op(self, left, right): - return left // right - - class Name(DslExpression): def pprint(self, indent=''): return self.name diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index d3e6fcf..ae0d3ae 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -4,8 +4,9 @@ from dateutil.relativedelta import relativedelta from scipy import array -from quantdsl.exceptions import DslSyntaxError -from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name, And, Or, Add, Sub, Mult, Div +from quantdsl.exceptions import DslSyntaxError, DslNameError +from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name, And, Or, Add, Sub, Mult, Div, Min, \ + Max, DslNamespace class Subclass(DslObject): @@ -273,3 +274,51 @@ def test_evaluate(self): obj = Div(Number(2.1), String('a')) with self.assertRaises(DslSyntaxError): obj.evaluate() + + +class TestMin(TestCase): + def test_evaluate(self): + obj = Min(Number(1), Number(2)) + self.assertEqual(obj.evaluate(), 1) + + obj = Min(Number(1), Number(array([1, 2, 3]))) + self.assertEqual(list(obj.evaluate()), list(array([1, 1, 1]))) + + obj = Min(Number(2), Number(array([1, 2, 3]))) + self.assertEqual(list(obj.evaluate()), list(array([1, 2, 2]))) + + obj = Min(Number(array([3, 2, 1])), Number(array([1, 2, 3]))) + self.assertEqual(list(obj.evaluate()), list(array([1, 2, 1]))) + + +class TestMax(TestCase): + def test_evaluate(self): + obj = Max(Number(1), Number(2)) + self.assertEqual(obj.evaluate(), 2) + + obj = Max(Number(1), Number(array([1, 2, 3]))) + self.assertEqual(list(obj.evaluate()), list(array([1, 2, 3]))) + + obj = Max(Number(2), Number(array([1, 2, 3]))) + self.assertEqual(list(obj.evaluate()), list(array([2, 2, 3]))) + + obj = Max(Number(array([3, 2, 1])), Number(array([1, 2, 3]))) + self.assertEqual(list(obj.evaluate()), list(array([3, 2, 3]))) + + +class TestName(TestCase): + def test_evaluate(self): + # Maybe if Name can take a string, perhaps also other things can? + # Maybe the parser should return Python string, numbers etc? + # Maybe String and Number etc don't add anything? + obj = Name('a') + self.assertEqual(obj.name, 'a') + + obj = Name(String('a')) + self.assertEqual(obj.name, 'a') + + with self.assertRaises(DslNameError): + obj.reduce(DslNamespace(), DslNamespace()) + + self.assertEqual(obj.reduce(DslNamespace({'a': 1}), DslNamespace()), Number(1)) + self.assertEqual(obj.reduce(DslNamespace({'a': datetime.timedelta(1)}), DslNamespace()), TimeDelta(datetime.timedelta(1))) \ No newline at end of file From 95bf7262053118f2f2dd14b4950df3580e741d98 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 19:27:02 +0100 Subject: [PATCH 14/52] Re-established purging of call results that have been fully used. Added class CallResultPolicy. Refactored usage counting. --- quantdsl/application/base.py | 47 +++------ quantdsl/application/call_result_policy.py | 78 +++++++++++++++ quantdsl/application/persistence_policy.py | 11 +++ quantdsl/application/with_multithreading.py | 7 +- quantdsl/domain/model/call_dependencies.py | 4 +- quantdsl/domain/model/call_result.py | 12 ++- .../domain/services/contract_valuations.py | 99 +++---------------- .../infrastructure/evaluation_subscriber.py | 6 +- quantdsl/tests/test_call_result_policy.py | 81 +++++++++++++++ quantdsl/tests/test_evaluation_subscriber.py | 14 +-- quantdsl/tests/test_semantics.py | 15 +-- 11 files changed, 230 insertions(+), 144 deletions(-) create mode 100644 quantdsl/application/call_result_policy.py create mode 100644 quantdsl/application/persistence_policy.py create mode 100644 quantdsl/tests/test_call_result_policy.py diff --git a/quantdsl/application/base.py b/quantdsl/application/base.py index fdc5d5e..0c3770d 100644 --- a/quantdsl/application/base.py +++ b/quantdsl/application/base.py @@ -1,18 +1,16 @@ -from abc import abstractmethod - import six from eventsourcing.application.base import EventSourcingApplication +from quantdsl.application.call_result_policy import CallResultPolicy +from quantdsl.application.persistence_policy import PersistencePolicy from quantdsl.domain.model.call_dependencies import register_call_dependencies from quantdsl.domain.model.call_dependents import register_call_dependents from quantdsl.domain.model.call_link import register_call_link -# from quantdsl.domain.model.call_requirement import register_call_requirement from quantdsl.domain.model.call_result import make_call_result_id from quantdsl.domain.model.contract_specification import ContractSpecification, register_contract_specification -from quantdsl.domain.model.contract_valuation import create_contract_valuation_id, start_contract_valuation -# from quantdsl.domain.model.dependency_graph import register_dependency_graph +from quantdsl.domain.model.contract_valuation import start_contract_valuation from quantdsl.domain.model.market_calibration import register_market_calibration -from quantdsl.domain.model.market_simulation import MarketSimulation, register_market_simulation +from quantdsl.domain.model.market_simulation import register_market_simulation from quantdsl.domain.model.perturbation_dependencies import PerturbationDependencies from quantdsl.domain.services.call_links import regenerate_execution_order from quantdsl.domain.services.contract_valuations import evaluate_call_and_queue_next_calls, loop_on_evaluation_queue @@ -53,7 +51,7 @@ class QuantDslApplication(EventSourcingApplication): Evaluate contract given call dependency graph and market simulation. """ - def __init__(self, call_evaluation_queue=None, result_counters=None, usage_counters=None, *args, **kwargs): + def __init__(self, call_evaluation_queue=None, result_counters=None, *args, **kwargs): super(QuantDslApplication, self).__init__(*args, **kwargs) self.contract_specification_repo = ContractSpecificationRepo(event_store=self.event_store, use_cache=True) self.contract_valuation_repo = ContractValuationRepo(event_store=self.event_store, use_cache=True) @@ -69,10 +67,10 @@ def __init__(self, call_evaluation_queue=None, result_counters=None, usage_count self.call_dependents_repo = CallDependentsRepo(event_store=self.event_store, use_cache=True) self.call_leafs_repo = CallLeafsRepo(event_store=self.event_store, use_cache=True) self.call_link_repo = CallLinkRepo(event_store=self.event_store, use_cache=True) - self.call_result_repo = CallResultRepo(event_store=self.event_store, use_cache=True) + # self.call_result_repo = CallResultRepo(event_store=self.event_store, use_cache=True) + self.call_result_repo = {} self.call_evaluation_queue = call_evaluation_queue self.result_counters = result_counters - self.usage_counters = usage_counters self.simulation_subscriber = SimulationSubscriber( market_calibration_repo=self.market_calibration_repo, @@ -96,25 +94,22 @@ def __init__(self, call_evaluation_queue=None, result_counters=None, usage_count call_evaluation_queue=self.call_evaluation_queue, call_leafs_repo=self.call_leafs_repo, result_counters=self.result_counters, - usage_counters=self.usage_counters, call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, ) + self.call_result_policy = CallResultPolicy(self.call_result_repo) + + def create_persistence_subscriber(self): + return PersistencePolicy(event_store=self.event_store) def close(self): self.evaluation_subscriber.close() self.dependency_graph_subscriber.close() self.simulation_subscriber.close() + self.call_result_policy.close() super(QuantDslApplication, self).close() - # Todo: Register historical data. - # def compute_market_calibration_params(self, price_process_name, historical_data): - # """ - # Returns market calibration params for given price process name and historical data. - # """ - # return compute_market_calibration_params(price_process_name, historical_data) - def register_contract_specification(self, source_code): """ Registers a new contract specification, from given Quant DSL source code. @@ -137,19 +132,6 @@ def register_market_simulation(self, market_calibration_id, observation_date, re return register_market_simulation(market_calibration_id, observation_date, requirements, path_count, interest_rate, perturbation_factor) - # def register_dependency_graph(self, contract_specification_id): - # return register_dependency_graph(contract_specification_id) - - # def register_call_requirement(self, call_id, dsl_source, effective_present_time): - # """ - # A call requirement is a node of the dependency graph. - # """ - # return register_call_requirement( - # call_id=call_id, - # dsl_source=dsl_source, - # effective_present_time=effective_present_time - # ) - def register_call_dependencies(self, call_id, dependencies): return register_call_dependencies(call_id=call_id, dependencies=dependencies) @@ -174,7 +156,7 @@ def start_contract_valuation(self, contract_specification_id, market_simulation_ assert isinstance(contract_specification_id, six.string_types), contract_specification_id return start_contract_valuation(contract_specification_id, market_simulation_id) - def loop_on_evaluation_queue(self, call_result_lock, compute_pool=None, result_counters=None, usage_counters=None): + def loop_on_evaluation_queue(self, call_result_lock, compute_pool=None, result_counters=None): loop_on_evaluation_queue( call_evaluation_queue=self.call_evaluation_queue, contract_valuation_repo=self.contract_valuation_repo, @@ -189,7 +171,6 @@ def loop_on_evaluation_queue(self, call_result_lock, compute_pool=None, result_c call_result_lock=call_result_lock, compute_pool=compute_pool, result_counters=result_counters, - usage_counters=usage_counters, ) def evaluate_call_and_queue_next_calls(self, contract_valuation_id, contract_specification_id, call_id, lock): @@ -249,6 +230,6 @@ def calc_call_costs(self, contract_specification_id): calls[call_id] = 1 else: assert isinstance(perturbation_dependencies, PerturbationDependencies) - # "1 + 2 * number of dependencies" because it does a double sided delta. + # "1 + 2 * number of dependencies" because of the double sided delta. calls[call_id] = 1 + 2 * len(perturbation_dependencies.dependencies) return calls diff --git a/quantdsl/application/call_result_policy.py b/quantdsl/application/call_result_policy.py new file mode 100644 index 0000000..4dfeee2 --- /dev/null +++ b/quantdsl/application/call_result_policy.py @@ -0,0 +1,78 @@ +from eventsourcing.domain.model.events import subscribe, unsubscribe + +from quantdsl.domain.model.call_dependencies import CallDependencies +from quantdsl.domain.model.call_dependents import CallDependents +from quantdsl.domain.model.call_result import CallResult, make_call_result_id +from quantdsl.infrastructure.event_sourced_repos.call_result_repo import CallResultRepo + + +class CallResultPolicy(object): + def __init__(self, call_result_repo): + self.call_result_repo = call_result_repo + self.result = {} + self.dependents = {} + self.dependencies = {} + self.usages = {} + + subscribe(self.is_call_dependencies_created, self.cache_dependencies) + subscribe(self.is_call_dependents_created, self.cache_dependents) + subscribe(self.is_call_result_created, self.cache_result) + subscribe(self.is_call_result_discarded, self.purge_result) + + def close(self): + unsubscribe(self.is_call_dependencies_created, self.cache_dependencies) + unsubscribe(self.is_call_dependents_created, self.cache_dependents) + unsubscribe(self.is_call_result_created, self.cache_result) + unsubscribe(self.is_call_result_discarded, self.purge_result) + + def is_call_dependencies_created(self, event): + return isinstance(event, CallDependencies.Created) + + def is_call_dependents_created(self, event): + return isinstance(event, CallDependents.Created) + + def is_call_result_created(self, event): + return isinstance(event, CallResult.Created) + + def is_call_result_discarded(self, event): + return isinstance(event, CallResult.Discarded) + + def cache_dependencies(self, event): + assert isinstance(event, CallDependencies.Created) + self.dependencies[event.entity_id] = event.dependencies[:] + + def cache_dependents(self, event): + assert isinstance(event, CallDependents.Created) + call_result_id = event.entity_id + self.dependents[call_result_id] = event.dependents[:] + self.usages[call_result_id] = [None] * (len(event.dependents) - 1) + + def cache_result(self, event): + assert isinstance(event, CallResult.Created) + + # Remember the call result entity. + this_result_id = event.entity_id + call_result = CallResult.mutator(event=event) + self.result[this_result_id] = call_result + if isinstance(self.call_result_repo, dict): + self.call_result_repo[this_result_id] = call_result + + # Remove one usage for each dependency of this result. + for dependency_id in self.dependencies.get(event.call_id, ()): + try: + self.usages[dependency_id].pop() + except IndexError: + # Discard the result when it has been fully used. + dependent_result_id = make_call_result_id(event.contract_valuation_id, dependency_id) + self.result[dependent_result_id].discard() + + def purge_result(self, event): + assert isinstance(event, CallResult.Discarded) + # Remove from the local results dict. + del (self.result[event.entity_id]) + + # Remove from the call result repo. + if isinstance(self.call_result_repo, dict): + del (self.call_result_repo[event.entity_id]) + elif isinstance(self.call_result_repo, CallResultRepo): + del (self.call_result_repo._cache[event.entity_id]) diff --git a/quantdsl/application/persistence_policy.py b/quantdsl/application/persistence_policy.py new file mode 100644 index 0000000..6be12fe --- /dev/null +++ b/quantdsl/application/persistence_policy.py @@ -0,0 +1,11 @@ +from eventsourcing.infrastructure.persistence_subscriber import PersistenceSubscriber + +from quantdsl.domain.model.call_result import CallResult + + +class PersistencePolicy(PersistenceSubscriber): + @staticmethod + def is_domain_event(event): + return PersistenceSubscriber.is_domain_event(event) and \ + not isinstance(event, (CallResult.Created, + CallResult.Discarded)) \ No newline at end of file diff --git a/quantdsl/application/with_multithreading.py b/quantdsl/application/with_multithreading.py index a83fc10..1ad9819 100644 --- a/quantdsl/application/with_multithreading.py +++ b/quantdsl/application/with_multithreading.py @@ -15,15 +15,16 @@ class QuantDslApplicationWithMultithreading(QuantDslApplication): def __init__(self, num_workers, *args, **kwargs): # Todo: Refactor the counter logic into a class, or find something that already does this. Perhaps use range. result_counters = {} - usage_counters = {} + # result_counters = None + super(QuantDslApplicationWithMultithreading, self).__init__(call_evaluation_queue=Queue(), result_counters=result_counters, - usage_counters=usage_counters, *args, **kwargs) + *args, **kwargs) call_result_lock = None compute_pool = get_compute_pool() # Start evaluation worker threads. - thread_target = lambda: self.loop_on_evaluation_queue(call_result_lock, compute_pool, result_counters, usage_counters) + thread_target = lambda: self.loop_on_evaluation_queue(call_result_lock, compute_pool, result_counters) for _ in range(num_workers): t = Thread(target=thread_target) t.setDaemon(True) diff --git a/quantdsl/domain/model/call_dependencies.py b/quantdsl/domain/model/call_dependencies.py index 8551dbe..864e6ff 100644 --- a/quantdsl/domain/model/call_dependencies.py +++ b/quantdsl/domain/model/call_dependencies.py @@ -8,7 +8,9 @@ class CallDependencies(EventSourcedEntity): """ class Created(EventSourcedEntity.Created): - pass + @property + def dependencies(self): + return self.__dict__['dependencies'] class Discarded(EventSourcedEntity.Discarded): pass diff --git a/quantdsl/domain/model/call_result.py b/quantdsl/domain/model/call_result.py index 6276a08..4daea4b 100644 --- a/quantdsl/domain/model/call_result.py +++ b/quantdsl/domain/model/call_result.py @@ -11,7 +11,17 @@ class CallResult(EventSourcedEntity): class Created(EventSourcedEntity.Created): - pass + @property + def call_id(self): + return self.__dict__['call_id'] + + @property + def contract_valuation_id(self): + return self.__dict__['contract_valuation_id'] + + @property + def result_value(self): + return self.__dict__['result_value'] class Discarded(EventSourcedEntity.Discarded): pass diff --git a/quantdsl/domain/services/contract_valuations.py b/quantdsl/domain/services/contract_valuations.py index c37926a..a778531 100644 --- a/quantdsl/domain/services/contract_valuations.py +++ b/quantdsl/domain/services/contract_valuations.py @@ -13,12 +13,13 @@ from quantdsl.domain.model.contract_valuation import ContractValuation from quantdsl.domain.services.call_links import regenerate_execution_order from quantdsl.domain.services.parser import dsl_parse +from quantdsl.infrastructure.event_sourced_repos.call_result_repo import CallResultRepo from quantdsl.semantics import DslNamespace def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, call_evaluation_queue, call_leafs_repo, call_link_repo, call_requirement_repo, call_result_repo, contract_valuation_repo, - market_simulation_repo, simulated_price_repo, result_counters, usage_counters, + market_simulation_repo, simulated_price_repo, result_counters, call_dependents_repo, perturbation_dependencies_repo, simulated_price_dependencies_repo): if not call_evaluation_queue: evaluate_contract_in_series( @@ -42,7 +43,6 @@ def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, c call_evaluation_queue=call_evaluation_queue, call_link_repo=call_link_repo, result_counters=result_counters, - usage_counters=usage_counters, call_dependencies_repo=call_dependencies_repo, call_dependents_repo=call_dependents_repo, perturbation_dependencies_repo=perturbation_dependencies_repo, @@ -120,7 +120,7 @@ def evaluate_contract_in_series(contract_valuation_id, contract_valuation_repo, def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo, call_leafs_repo, call_link_repo, - call_evaluation_queue, result_counters, usage_counters, call_dependencies_repo, + call_evaluation_queue, result_counters, call_dependencies_repo, call_dependents_repo, perturbation_dependencies_repo, simulated_price_dependencies_repo): """ @@ -134,18 +134,15 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo contract_specification_id = contract_valuation.contract_specification_id if result_counters is not None: - assert usage_counters is not None for call_id in regenerate_execution_order(contract_specification_id, call_link_repo): call_dependencies = call_dependencies_repo[call_id] call_dependents = call_dependents_repo[call_id] assert isinstance(call_dependencies, CallDependencies) assert isinstance(call_dependents, CallDependents) count_dependencies = len(call_dependencies.dependencies) - count_dependents = len(call_dependents.dependents) # Crude attempt to count down using atomic operations, so we get an exception when we can't pop off the last one. call_result_id = make_call_result_id(contract_valuation_id, call_id) result_counters[call_result_id] = [None] * (count_dependencies - 1) - usage_counters[call_result_id] = [None] * (count_dependents - 1) call_leafs = call_leafs_repo[contract_specification_id] # assert isinstance(call_leafs, CallLeafs) @@ -157,7 +154,7 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, call_requirement_repo, market_simulation_repo, call_dependencies_repo, call_result_repo, simulated_price_repo, call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo, - call_result_lock, compute_pool=None, result_counters=None, usage_counters=None): + call_result_lock, compute_pool=None, result_counters=None): while True: item = call_evaluation_queue.get() try: @@ -180,7 +177,6 @@ def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, cal call_result_lock=call_result_lock, compute_pool=compute_pool, result_counters=result_counters, - usage_counters=usage_counters, ) finally: call_evaluation_queue.task_done() @@ -191,7 +187,7 @@ def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specifica call_dependencies_repo, call_result_repo, simulated_price_repo, call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo, call_result_lock, - compute_pool=None, result_counters=None, usage_counters=None): + compute_pool=None, result_counters=None): # Get the contract valuation. contract_valuation = contract_valuation_repo[contract_valuation_id] @@ -255,24 +251,6 @@ def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specifica result_counters=result_counters, ) - # Check requirements, and discard result when dependency has been fully used. - if usage_counters is not None: - call_dependencies = call_dependencies_repo[call_id] - assert isinstance(call_dependencies, CallDependencies) - for dependency_id in call_dependencies.dependencies: - dependency_result_id = make_call_result_id(contract_valuation_id, dependency_id) - try: - usage_counters[dependency_result_id].pop() # Pop one off the array (atomic decrement). - except (KeyError, IndexError): - call_result = call_result_repo[dependency_result_id] - # Todo: Maybe do discard operations after lock has been released? - call_result.discard() - # Need to remove from the cache if we are to save memory. - try: - del(call_result_repo._cache[dependency_result_id]) - except: - pass - if call_result_lock is not None: # Make a list from the generator, if we are locking results. next_call_ids = list(ready_generator) @@ -299,69 +277,24 @@ def find_dependents_ready_to_be_evaluated(contract_valuation_id, call_id, call_d # assert isinstance(call_dependents_repo, CallDependentsRepository) # assert isinstance(call_dependencies_repo, CallDependenciesRepository) # assert isinstance(call_result_repo, CallResultRepository) + assert result_counters is not None # Get dependents (if any exist). try: call_dependents = call_dependents_repo[call_id] - - # Don't worry if there are none. except KeyError: + # Don't worry if there are none. pass - else: - # Check if any dependents are ready to be evaluated. - ready_dependents = [] - if result_counters is not None: - for dependent_id in call_dependents.dependents: - dependent_result_id = make_call_result_id(contract_valuation_id, dependent_id) - try: - result_counters[dependent_result_id].pop() # Pop one off the array (atomic decrement). - except (KeyError, IndexError): - ready_dependents.append(dependent_id) - return ready_dependents - - # assert isinstance(call_dependents, CallDependents) - - else: - # Todo: Maybe speed this up by prepreparing the dependent-requirements (so it's just one query). - dependent_threads = [] - for dependent_id in call_dependents.dependents: - # Single-threaded identification of dependents ready to be evaluated. - add_dependent_if_ready(call_dependencies_repo, - call_id, - call_result_repo, - contract_valuation_id, - dependent_id, - ready_dependents) - [t.join() for t in dependent_threads] - return ready_dependents - - -def add_dependent_if_ready(call_dependencies_repo, call_id, call_result_repo, contract_valuation_id, - dependent_id, ready_dependents): - - # Get the requirements of this dependent. - dependent_dependencies = call_dependencies_repo[dependent_id] - assert isinstance(dependent_dependencies, CallDependencies) - - for dependent_dependency_id in dependent_dependencies.dependencies: - - # Skip if this dependent dependency is the given call. - if dependent_dependency_id == call_id: - continue - - # Skip if a result is missing. - if is_result_missing(contract_valuation_id, dependent_dependency_id, call_result_repo): - break - - else: - ready_dependents.append(dependent_id) - - -def is_result_missing(contract_valuation_id, dependent_dependency_id, call_result_repo): - call_result_id = make_call_result_id(contract_valuation_id, dependent_dependency_id) - return call_result_id not in call_result_repo + ready = [] + for dependent_id in call_dependents.dependents: + dependent_result_id = make_call_result_id(contract_valuation_id, dependent_id) + try: + result_counters[dependent_result_id].pop() # Pop one off the array (atomic decrement). + except (KeyError, IndexError): + ready.append(dependent_id) + return ready def compute_call_result(contract_valuation, call_requirement, market_simulation, perturbation_dependencies, @@ -511,7 +444,7 @@ def get_compute_pool(): def get_dependency_results(contract_valuation_id, call_id, dependencies_repo, result_repo): - assert isinstance(result_repo, CallResultRepository), result_repo + assert isinstance(result_repo, (CallResultRepository, dict)), result_repo dependency_results = {} stub_dependencies = dependencies_repo[call_id] assert isinstance(stub_dependencies, CallDependencies), stub_dependencies diff --git a/quantdsl/infrastructure/evaluation_subscriber.py b/quantdsl/infrastructure/evaluation_subscriber.py index d31026f..9946d18 100644 --- a/quantdsl/infrastructure/evaluation_subscriber.py +++ b/quantdsl/infrastructure/evaluation_subscriber.py @@ -18,13 +18,13 @@ class EvaluationSubscriber(object): def __init__(self, contract_valuation_repo, call_link_repo, call_dependencies_repo, call_requirement_repo, call_result_repo, simulated_price_repo, market_simulation_repo, call_leafs_repo, - call_evaluation_queue, result_counters, usage_counters, call_dependents_repo, + call_evaluation_queue, result_counters, call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo): assert isinstance(contract_valuation_repo, ContractValuationRepository), contract_valuation_repo assert isinstance(call_link_repo, CallLinkRepository), call_link_repo assert isinstance(call_dependencies_repo, CallDependenciesRepository), call_dependencies_repo assert isinstance(call_requirement_repo, CallRequirementRepository), call_requirement_repo - assert isinstance(call_result_repo, CallResultRepository), call_result_repo + assert isinstance(call_result_repo, (CallResultRepository, dict)), call_result_repo assert isinstance(simulated_price_repo, SimulatedPriceRepository), simulated_price_repo assert isinstance(market_simulation_repo, MarketSimulationRepository), market_simulation_repo assert isinstance(call_dependents_repo, CallDependentsRepository), call_dependents_repo @@ -41,7 +41,6 @@ def __init__(self, contract_valuation_repo, call_link_repo, call_dependencies_re self.call_leafs_repo = call_leafs_repo self.call_evaluation_queue = call_evaluation_queue self.result_counters = result_counters - self.usage_counters = usage_counters self.call_dependents_repo = call_dependents_repo self.perturbation_dependencies_repo = perturbation_dependencies_repo self.simulated_price_dependencies_repo = simulated_price_requirements_repo @@ -66,7 +65,6 @@ def generate_contract_valuation(self, event): market_simulation_repo=self.market_simulation_repo, simulated_price_repo=self.simulated_price_repo, result_counters=self.result_counters, - usage_counters=self.usage_counters, call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_dependencies_repo=self.simulated_price_dependencies_repo, diff --git a/quantdsl/tests/test_call_result_policy.py b/quantdsl/tests/test_call_result_policy.py new file mode 100644 index 0000000..013286a --- /dev/null +++ b/quantdsl/tests/test_call_result_policy.py @@ -0,0 +1,81 @@ +from unittest.case import TestCase + +from eventsourcing.domain.model.events import assert_event_handlers_empty +from eventsourcing.infrastructure.event_store import EventStore +from eventsourcing.infrastructure.persistence_subscriber import PersistenceSubscriber +from eventsourcing.infrastructure.stored_events.python_objects_stored_events import PythonObjectsStoredEventRepository + +from quantdsl.application.call_result_policy import CallResultPolicy +from quantdsl.domain.model.call_dependencies import register_call_dependencies +from quantdsl.domain.model.call_dependents import register_call_dependents +from quantdsl.domain.model.call_result import register_call_result +from quantdsl.infrastructure.event_sourced_repos.call_dependencies_repo import CallDependenciesRepo +from quantdsl.infrastructure.event_sourced_repos.call_dependents_repo import CallDependentsRepo +from quantdsl.infrastructure.event_sourced_repos.call_requirement_repo import CallRequirementRepo + + +class TestCallResultPolicy(TestCase): + def setUp(self): + assert_event_handlers_empty() + self.es = EventStore(PythonObjectsStoredEventRepository()) + self.ps = PersistenceSubscriber(self.es) + # self.call_result_repo = CallResultRepo(self.es) + self.call_result_repo = {} + self.call_dependencies_repo = CallDependenciesRepo(self.es) + self.call_dependents_repo = CallDependentsRepo(self.es) + self.call_requirement_repo = CallRequirementRepo(self.es) + self.policy = CallResultPolicy(call_result_repo=self.call_result_repo) + + def tearDown(self): + self.ps.close() + self.policy.close() + assert_event_handlers_empty() + + def test_delete_result(self): + # In this test, there are two "calls": call1 and call2. + # It is supposed that call1 happens first, and call2 uses the result of call1. + # Therefore call2 depends upon call1, call1 is a dependency of call2, and call2 is a dependent of call1. + call1_id = 'call1' + call2_id = 'call2' + contract_valuation_id = 'val1' + contract_specification_id = 'spec1' + # call1_id = uuid4().hex + # call2_id = uuid4().hex + # contract_valuation_id = uuid4().hex + # contract_specification_id = uuid4().hex + + register_call_dependencies(call2_id, [call1_id]) + + # Check the policy has the dependencies for call2. + self.assertEqual(self.policy.dependencies[call2_id], [call1_id]) + + # Register dependents of call1, as call2. + register_call_dependents(call1_id, [call2_id]) + + # Check the policy has the dependencies for call2. + self.assertEqual(self.policy.dependents[call1_id], [call2_id]) + + # Register call result for call1. + # - this should trigger deletion of call2 result + call1_result = register_call_result(call1_id, 1.0, {}, contract_valuation_id, contract_specification_id) + + # Check the policy has the result for call1. + self.assertTrue(call1_result.id in self.policy.result) + + # Check the result for call1 exists. + self.assertTrue(call1_result.id in self.call_result_repo) + + # Register call result for call2. + call2_result = register_call_result(call2_id, 1.0, {}, contract_valuation_id, contract_specification_id) + + # Check the policy has the result for call2. + self.assertTrue(call2_result.id in self.policy.result) + + # Check the result for call2 exists. + self.assertTrue(call2_result.id in self.call_result_repo) + + # Check the policy does not have the result for call1. + self.assertFalse(call1_result.id in self.policy.result) + + # Check the result for call1 doesn't exist (because it's dependents have results). + self.assertFalse(call1_result.id in self.call_result_repo) diff --git a/quantdsl/tests/test_evaluation_subscriber.py b/quantdsl/tests/test_evaluation_subscriber.py index 51ec975..7c2e942 100644 --- a/quantdsl/tests/test_evaluation_subscriber.py +++ b/quantdsl/tests/test_evaluation_subscriber.py @@ -1,18 +1,16 @@ -import datetime import unittest -from eventsourcing.domain.model.events import publish, assert_event_handlers_empty -from mock import MagicMock, Mock -from mock import patch +from eventsourcing.domain.model.events import assert_event_handlers_empty, publish +from mock import MagicMock, Mock, patch from quantdsl.domain.model.call_dependencies import CallDependenciesRepository from quantdsl.domain.model.call_dependents import CallDependentsRepository -from quantdsl.domain.model.call_leafs import CallLeafsRepository, CallLeafs +from quantdsl.domain.model.call_leafs import CallLeafs, CallLeafsRepository from quantdsl.domain.model.call_link import CallLinkRepository from quantdsl.domain.model.call_requirement import CallRequirementRepository from quantdsl.domain.model.call_result import CallResultRepository -from quantdsl.domain.model.contract_valuation import ContractValuationRepository, ContractValuation -from quantdsl.domain.model.market_simulation import MarketSimulationRepository, MarketSimulation +from quantdsl.domain.model.contract_valuation import ContractValuation, ContractValuationRepository +from quantdsl.domain.model.market_simulation import MarketSimulation, MarketSimulationRepository from quantdsl.domain.model.simulated_price import SimulatedPriceRepository from quantdsl.infrastructure.evaluation_subscriber import EvaluationSubscriber from quantdsl.infrastructure.event_sourced_repos.perturbation_dependencies_repo import PerturbationDependenciesRepo @@ -21,7 +19,6 @@ class TestEvaluationSubscriber(unittest.TestCase): - def setUp(self): assert_event_handlers_empty() contract_valuation_repo = MagicMock(spec=ContractValuationRepository) @@ -45,7 +42,6 @@ def setUp(self): call_leafs_repo=call_leafs_repo, call_evaluation_queue=None, result_counters=None, - usage_counters=None, call_dependents_repo=call_dependents_repo, perturbation_dependencies_repo=perturbation_dependencies_repo, simulated_price_requirements_repo=simulated_price_requirements_repo diff --git a/quantdsl/tests/test_semantics.py b/quantdsl/tests/test_semantics.py index ae0d3ae..0122d60 100644 --- a/quantdsl/tests/test_semantics.py +++ b/quantdsl/tests/test_semantics.py @@ -4,9 +4,9 @@ from dateutil.relativedelta import relativedelta from scipy import array -from quantdsl.exceptions import DslSyntaxError, DslNameError -from quantdsl.semantics import Date, DslObject, Number, String, TimeDelta, Name, And, Or, Add, Sub, Mult, Div, Min, \ - Max, DslNamespace +from quantdsl.exceptions import DslNameError, DslSyntaxError +from quantdsl.semantics import Add, And, Date, Div, DslNamespace, DslObject, Max, Min, Mult, Name, Number, Or, \ + String, Sub, TimeDelta class Subclass(DslObject): @@ -164,7 +164,6 @@ def test_str(self): class TestAnd(TestCase): - def test_evaluate(self): obj = And([Number(1), Number(1)]) self.assertTrue(obj.evaluate()) @@ -184,7 +183,6 @@ def test_str(self): class TestOr(TestCase): - def test_evaluate(self): obj = Or([Number(1), Number(1)]) self.assertTrue(obj.evaluate()) @@ -204,7 +202,6 @@ def test_str(self): class TestAndOr(TestCase): - def test_str(self): obj = And([Number(1), Or([Number(2), Number(3)])]) self.assertEqual(str(obj), '(1 and (2 or 3))') @@ -213,7 +210,6 @@ def test_str(self): class TestAdd(TestCase): - def test_evaluate(self): obj = Add(Number(1), Number(1)) self.assertEqual(obj.evaluate(), 2) @@ -227,7 +223,6 @@ def test_evaluate(self): class TestSub(TestCase): - def test_evaluate(self): obj = Sub(Number(1), Number(1)) self.assertEqual(obj.evaluate(), 0) @@ -238,7 +233,6 @@ def test_evaluate(self): class TestMul(TestCase): - def test_evaluate(self): obj = Mult(Number(2), Number(2)) self.assertEqual(obj.evaluate(), 4) @@ -321,4 +315,5 @@ def test_evaluate(self): obj.reduce(DslNamespace(), DslNamespace()) self.assertEqual(obj.reduce(DslNamespace({'a': 1}), DslNamespace()), Number(1)) - self.assertEqual(obj.reduce(DslNamespace({'a': datetime.timedelta(1)}), DslNamespace()), TimeDelta(datetime.timedelta(1))) \ No newline at end of file + self.assertEqual(obj.reduce(DslNamespace({'a': datetime.timedelta(1)}), DslNamespace()), + TimeDelta(datetime.timedelta(1))) From f58334e1722666a95cf226085dba35a6ff5a3ce7 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 20:32:42 +0100 Subject: [PATCH 15/52] Refactored call propagation via result counting. Also removed more unused stuff. --- quantdsl/application/__init__.py | 1 - quantdsl/application/base.py | 11 +- quantdsl/application/call_result_policy.py | 18 +- quantdsl/application/with_multithreading.py | 21 +-- quantdsl/domain/model/__init__.py | 1 - quantdsl/domain/model/call_result.py | 17 +- .../domain/services/contract_valuations.py | 158 ++++-------------- quantdsl/domain/services/dependency_graphs.py | 2 +- quantdsl/domain/services/uuids.py | 1 - quantdsl/tests/test_application.py | 33 ++-- quantdsl/tests/test_multiprocessing.py | 74 -------- 11 files changed, 78 insertions(+), 259 deletions(-) delete mode 100644 quantdsl/tests/test_multiprocessing.py diff --git a/quantdsl/application/__init__.py b/quantdsl/application/__init__.py index 6ccd6fe..e69de29 100644 --- a/quantdsl/application/__init__.py +++ b/quantdsl/application/__init__.py @@ -1 +0,0 @@ -__author__ = 'john' diff --git a/quantdsl/application/base.py b/quantdsl/application/base.py index 0c3770d..b996fa3 100644 --- a/quantdsl/application/base.py +++ b/quantdsl/application/base.py @@ -22,7 +22,6 @@ from quantdsl.infrastructure.event_sourced_repos.call_leafs_repo import CallLeafsRepo from quantdsl.infrastructure.event_sourced_repos.call_link_repo import CallLinkRepo from quantdsl.infrastructure.event_sourced_repos.call_requirement_repo import CallRequirementRepo -from quantdsl.infrastructure.event_sourced_repos.call_result_repo import CallResultRepo from quantdsl.infrastructure.event_sourced_repos.contract_specification_repo import ContractSpecificationRepo from quantdsl.infrastructure.event_sourced_repos.contract_valuation_repo import ContractValuationRepo from quantdsl.infrastructure.event_sourced_repos.market_calibration_repo import MarketCalibrationRepo @@ -98,7 +97,7 @@ def __init__(self, call_evaluation_queue=None, result_counters=None, *args, **kw perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, ) - self.call_result_policy = CallResultPolicy(self.call_result_repo) + self.call_result_policy = CallResultPolicy(self.call_result_repo, self.call_evaluation_queue) def create_persistence_subscriber(self): return PersistencePolicy(event_store=self.event_store) @@ -156,7 +155,7 @@ def start_contract_valuation(self, contract_specification_id, market_simulation_ assert isinstance(contract_specification_id, six.string_types), contract_specification_id return start_contract_valuation(contract_specification_id, market_simulation_id) - def loop_on_evaluation_queue(self, call_result_lock, compute_pool=None, result_counters=None): + def loop_on_evaluation_queue(self, compute_pool=None): loop_on_evaluation_queue( call_evaluation_queue=self.call_evaluation_queue, contract_valuation_repo=self.contract_valuation_repo, @@ -168,17 +167,14 @@ def loop_on_evaluation_queue(self, call_result_lock, compute_pool=None, result_c call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, - call_result_lock=call_result_lock, compute_pool=compute_pool, - result_counters=result_counters, ) - def evaluate_call_and_queue_next_calls(self, contract_valuation_id, contract_specification_id, call_id, lock): + def evaluate_call_and_queue_next_calls(self, contract_valuation_id, contract_specification_id, call_id): evaluate_call_and_queue_next_calls( contract_valuation_id=contract_valuation_id, contract_specification_id=contract_specification_id, call_id=call_id, - call_evaluation_queue=self.call_evaluation_queue, contract_valuation_repo=self.contract_valuation_repo, call_requirement_repo=self.call_requirement_repo, market_simulation_repo=self.market_simulation_repo, @@ -188,7 +184,6 @@ def evaluate_call_and_queue_next_calls(self, contract_valuation_id, contract_spe call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, - call_result_lock=lock, ) def compile(self, source_code): diff --git a/quantdsl/application/call_result_policy.py b/quantdsl/application/call_result_policy.py index 4dfeee2..93d0472 100644 --- a/quantdsl/application/call_result_policy.py +++ b/quantdsl/application/call_result_policy.py @@ -7,12 +7,14 @@ class CallResultPolicy(object): - def __init__(self, call_result_repo): + def __init__(self, call_result_repo, call_evaluation_queue=None): self.call_result_repo = call_result_repo + self.call_evaluation_queue = call_evaluation_queue self.result = {} self.dependents = {} self.dependencies = {} self.usages = {} + self.result_counters = {} subscribe(self.is_call_dependencies_created, self.cache_dependencies) subscribe(self.is_call_dependents_created, self.cache_dependents) @@ -39,7 +41,9 @@ def is_call_result_discarded(self, event): def cache_dependencies(self, event): assert isinstance(event, CallDependencies.Created) - self.dependencies[event.entity_id] = event.dependencies[:] + call_result_id = event.entity_id + self.dependencies[call_result_id] = event.dependencies[:] + self.result_counters[call_result_id] = [None] * (len(event.dependencies) - 1) def cache_dependents(self, event): assert isinstance(event, CallDependents.Created) @@ -66,6 +70,16 @@ def cache_result(self, event): dependent_result_id = make_call_result_id(event.contract_valuation_id, dependency_id) self.result[dependent_result_id].discard() + # Remove one result for each dependent of this result. + if self.call_evaluation_queue: + for dependent_id in self.dependents.get(event.call_id, ()): + try: + self.result_counters[dependent_id].pop() + except IndexError: + # Queue the call if a dependent has all its results. + job = (event.contract_specification_id, event.contract_valuation_id, dependent_id) + self.call_evaluation_queue.put(job) + def purge_result(self, event): assert isinstance(event, CallResult.Discarded) # Remove from the local results dict. diff --git a/quantdsl/application/with_multithreading.py b/quantdsl/application/with_multithreading.py index 1ad9819..257e8ec 100644 --- a/quantdsl/application/with_multithreading.py +++ b/quantdsl/application/with_multithreading.py @@ -1,11 +1,5 @@ -from quantdsl.domain.services.contract_valuations import get_compute_pool - -try: - from queue import Queue -except ImportError: - from Queue import Queue - from threading import Thread +import six.moves.queue as queue from quantdsl.application.base import QuantDslApplication @@ -13,19 +7,10 @@ class QuantDslApplicationWithMultithreading(QuantDslApplication): def __init__(self, num_workers, *args, **kwargs): - # Todo: Refactor the counter logic into a class, or find something that already does this. Perhaps use range. - result_counters = {} - # result_counters = None - - super(QuantDslApplicationWithMultithreading, self).__init__(call_evaluation_queue=Queue(), - result_counters=result_counters, + super(QuantDslApplicationWithMultithreading, self).__init__(call_evaluation_queue=queue.LifoQueue(), *args, **kwargs) - call_result_lock = None - compute_pool = get_compute_pool() - # Start evaluation worker threads. - thread_target = lambda: self.loop_on_evaluation_queue(call_result_lock, compute_pool, result_counters) for _ in range(num_workers): - t = Thread(target=thread_target) + t = Thread(target=self.loop_on_evaluation_queue) t.setDaemon(True) t.start() diff --git a/quantdsl/domain/model/__init__.py b/quantdsl/domain/model/__init__.py index 6ccd6fe..e69de29 100644 --- a/quantdsl/domain/model/__init__.py +++ b/quantdsl/domain/model/__init__.py @@ -1 +0,0 @@ -__author__ = 'john' diff --git a/quantdsl/domain/model/call_result.py b/quantdsl/domain/model/call_result.py index 4daea4b..66ae142 100644 --- a/quantdsl/domain/model/call_result.py +++ b/quantdsl/domain/model/call_result.py @@ -1,15 +1,9 @@ -# from multiprocessing.sharedctypes import SynchronizedArray -from threading import Lock - import scipy -from eventsourcing.domain.model.entity import EventSourcedEntity, EntityRepository +from eventsourcing.domain.model.entity import EntityRepository, EventSourcedEntity from eventsourcing.domain.model.events import publish -# from quantdsl.semantics import numpy_from_sharedmem - class CallResult(EventSourcedEntity): - class Created(EventSourcedEntity.Created): @property def call_id(self): @@ -19,6 +13,10 @@ def call_id(self): def contract_valuation_id(self): return self.__dict__['contract_valuation_id'] + @property + def contract_specification_id(self): + return self.__dict__['contract_specification_id'] + @property def result_value(self): return self.__dict__['result_value'] @@ -26,7 +24,8 @@ def result_value(self): class Discarded(EventSourcedEntity.Discarded): pass - def __init__(self, result_value, perturbed_values, contract_valuation_id, call_id, contract_specification_id, **kwargs): + def __init__(self, result_value, perturbed_values, contract_valuation_id, call_id, contract_specification_id, + **kwargs): super(CallResult, self).__init__(**kwargs) self._result_value = result_value self._perturbed_values = perturbed_values @@ -57,8 +56,6 @@ def contract_specification_id(self): @property def scalar_result_value(self): result_value = self._result_value - # if isinstance(result_value, SynchronizedArray): - # result_value = numpy_from_sharedmem(result_value) if isinstance(result_value, scipy.ndarray): result_value = result_value.mean() return result_value diff --git a/quantdsl/domain/services/contract_valuations.py b/quantdsl/domain/services/contract_valuations.py index a778531..d829d96 100644 --- a/quantdsl/domain/services/contract_valuations.py +++ b/quantdsl/domain/services/contract_valuations.py @@ -5,22 +5,22 @@ from quantdsl.domain.model.call_dependencies import CallDependencies from quantdsl.domain.model.call_dependents import CallDependents from quantdsl.domain.model.call_requirement import CallRequirement -from quantdsl.domain.model.call_result import register_call_result, make_call_result_id, CallResult, \ - CallResultRepository, ResultValueComputed +from quantdsl.domain.model.call_result import CallResult, CallResultRepository, ResultValueComputed, \ + make_call_result_id, register_call_result +from quantdsl.domain.model.contract_valuation import ContractValuation from quantdsl.domain.model.market_simulation import MarketSimulation from quantdsl.domain.model.perturbation_dependencies import PerturbationDependencies from quantdsl.domain.model.simulated_price import make_simulated_price_id -from quantdsl.domain.model.contract_valuation import ContractValuation from quantdsl.domain.services.call_links import regenerate_execution_order from quantdsl.domain.services.parser import dsl_parse -from quantdsl.infrastructure.event_sourced_repos.call_result_repo import CallResultRepo from quantdsl.semantics import DslNamespace def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, call_evaluation_queue, call_leafs_repo, call_link_repo, call_requirement_repo, call_result_repo, contract_valuation_repo, market_simulation_repo, simulated_price_repo, result_counters, - call_dependents_repo, perturbation_dependencies_repo, simulated_price_dependencies_repo): + call_dependents_repo, perturbation_dependencies_repo, + simulated_price_dependencies_repo): if not call_evaluation_queue: evaluate_contract_in_series( contract_valuation_id=contract_valuation_id, @@ -103,21 +103,6 @@ def evaluate_contract_in_series(contract_valuation_id, contract_valuation_repo, contract_specification_id=contract_specification_id, ) - # # Check for results that should be deleted. - # # - dependency results should be deleted if there is a result for each dependent of the dependency - # call_dependencies = call_dependencies_repo[call_id] - # assert isinstance(call_dependencies, CallDependencies) - # call_dependents = call_dependents_repo[call_id] - # assert isinstance(call_dependents, CallDependents), (type(call_dependents), CallDependents) - # - # for dependency_id in call_dependencies.dependencies: - # for dependent_id in call_dependents.dependents[dependency_id]: - # if dependent_id != call_id and dependent_id not in call_result_repo: - # # Need to keep it. - # break - # else: - # del (call_result_repo[dependency_id]) - def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo, call_leafs_repo, call_link_repo, call_evaluation_queue, result_counters, call_dependencies_repo, @@ -140,7 +125,8 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo assert isinstance(call_dependencies, CallDependencies) assert isinstance(call_dependents, CallDependents) count_dependencies = len(call_dependencies.dependencies) - # Crude attempt to count down using atomic operations, so we get an exception when we can't pop off the last one. + # Crude attempt to count down using atomic operations, so we get an exception when we can't pop off the + # last one. call_result_id = make_call_result_id(contract_valuation_id, call_id) result_counters[call_result_id] = [None] * (count_dependencies - 1) @@ -154,7 +140,7 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, call_requirement_repo, market_simulation_repo, call_dependencies_repo, call_result_repo, simulated_price_repo, call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo, - call_result_lock, compute_pool=None, result_counters=None): + compute_pool=None): while True: item = call_evaluation_queue.get() try: @@ -164,7 +150,6 @@ def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, cal contract_valuation_id=contract_valuation_id, contract_specification_id=contract_specification_id, call_id=call_id, - call_evaluation_queue=call_evaluation_queue, contract_valuation_repo=contract_valuation_repo, call_requirement_repo=call_requirement_repo, market_simulation_repo=market_simulation_repo, @@ -174,21 +159,18 @@ def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, cal call_dependents_repo=call_dependents_repo, perturbation_dependencies_repo=perturbation_dependencies_repo, simulated_price_requirements_repo=simulated_price_requirements_repo, - call_result_lock=call_result_lock, compute_pool=compute_pool, - result_counters=result_counters, ) finally: call_evaluation_queue.task_done() -def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specification_id, call_id, call_evaluation_queue, +def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specification_id, call_id, contract_valuation_repo, call_requirement_repo, market_simulation_repo, call_dependencies_repo, call_result_repo, simulated_price_repo, call_dependents_repo, perturbation_dependencies_repo, - simulated_price_requirements_repo, call_result_lock, - compute_pool=None, result_counters=None): - + simulated_price_requirements_repo, + compute_pool=None): # Get the contract valuation. contract_valuation = contract_valuation_repo[contract_valuation_id] assert isinstance(contract_valuation, ContractValuation) @@ -196,78 +178,38 @@ def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specifica # Get the market simulation. market_simulation = market_simulation_repo[contract_valuation.market_simulation_id] + # Get the call requirement. call_requirement = call_requirement_repo[call_id] + # Get the perturbation dependencies for this call. perturbation_dependencies = perturbation_dependencies_repo[call_id] assert isinstance(perturbation_dependencies, PerturbationDependencies) # Get the simulated price requirements for this call. simulation_requirements = simulated_price_requirements_repo[call_id].requirements + # Compute the call result. result_value, perturbed_values = compute_call_result( - contract_valuation=contract_valuation, - call_requirement=call_requirement, - market_simulation=market_simulation, - perturbation_dependencies=perturbation_dependencies, - call_dependencies_repo=call_dependencies_repo, - call_result_repo=call_result_repo, - simulated_price_repo=simulated_price_repo, - perturbation_dependencies_repo=perturbation_dependencies_repo, - simulation_requirements=simulation_requirements, - compute_pool=compute_pool, - ) - - # Lock the results. - # - avoids race conditions when checking, after a result - # has been written, if all results are now available, whilst - # others are writing results. - # - could perhaps do this with optimistic concurrency control, so - # that result events can be collected by dependents - # and then evaluated when all are received - to be robust against - # concurrent operations causing concurrency exceptions, an accumulating - # operation would require to be retried only as many times as there are - # remaining dependents. - if call_result_lock is not None: - call_result_lock.acquire() - - try: - # Register this result. - # Todo: Retries on concurrency errors. - register_call_result( - call_id=call_id, - result_value=result_value, - perturbed_values=perturbed_values, - contract_valuation_id=contract_valuation_id, - contract_specification_id=contract_specification_id, - ) - - # Find next calls. - ready_generator = find_dependents_ready_to_be_evaluated( - contract_valuation_id=contract_valuation_id, - call_id=call_id, - call_dependencies_repo=call_dependencies_repo, - call_dependents_repo=call_dependents_repo, - call_result_repo=call_result_repo, - result_counters=result_counters, - ) - - if call_result_lock is not None: - # Make a list from the generator, if we are locking results. - next_call_ids = list(ready_generator) - else: - # Otherwise put things directly on the queue. - next_call_ids = [] - for next_call_id in ready_generator: - call_evaluation_queue.put((contract_specification_id, contract_valuation_id, next_call_id)) - - finally: - # Unlock the results. - if call_result_lock is not None: - call_result_lock.release() + contract_valuation=contract_valuation, + call_requirement=call_requirement, + market_simulation=market_simulation, + perturbation_dependencies=perturbation_dependencies, + call_dependencies_repo=call_dependencies_repo, + call_result_repo=call_result_repo, + simulated_price_repo=simulated_price_repo, + perturbation_dependencies_repo=perturbation_dependencies_repo, + simulation_requirements=simulation_requirements, + compute_pool=compute_pool, + ) - # Queue the next calls (if there are any - see above). - for next_call_id in next_call_ids: - call_evaluation_queue.put((contract_specification_id, contract_valuation_id, next_call_id)) + # Register the call result. + register_call_result( + call_id=call_id, + result_value=result_value, + perturbed_values=perturbed_values, + contract_valuation_id=contract_valuation_id, + contract_specification_id=contract_specification_id, + ) def find_dependents_ready_to_be_evaluated(contract_valuation_id, call_id, call_dependents_repo, call_dependencies_repo, @@ -324,23 +266,6 @@ def compute_call_result(contract_valuation, call_requirement, market_simulation, present_time = call_requirement.effective_present_time or market_simulation.observation_date simulated_value_dict = {} - # market_simulation.dependencies - - # - # all_fixing_dates = set([present_time] + list_fixing_dates(dsl_expr)) - # market_dependencies = perturbation_dependencies_repo[call_requirement.id] - # assert isinstance(market_dependencies, PerturbationDependencies) - # all_delivery_points = market_dependencies.dependencies - # for fixing_date in all_fixing_dates: - # for delivery_point in all_delivery_points: - # market_name = delivery_point[0] - # delivery_date = delivery_point[1] - # simulation_id = contract_valuation.market_simulation_id - # price_id = make_simulated_price_id(simulation_id, market_name, fixing_date, delivery_date) - # simulated_price = simulated_price_dict[price_id] - # # assert isinstance(simulated_price, SimulatedPrice) - # # assert isinstance(simulated_price.value, scipy.ndarray) - # simulated_value_dict[price_id] = simulated_price.value simulation_id = contract_valuation.market_simulation_id @@ -365,8 +290,9 @@ def compute_call_result(contract_valuation, call_requirement, market_simulation, # Compute the call result. if compute_pool is None: result_value, perturbed_values = evaluate_dsl_expr(dsl_expr, first_commodity_name, market_simulation.id, - market_simulation.interest_rate, present_time, simulated_value_dict, - perturbation_dependencies.dependencies, + market_simulation.interest_rate, present_time, + simulated_value_dict, + perturbation_dependencies.dependencies, dependency_results, market_simulation.path_count, market_simulation.perturbation_factor) else: @@ -385,7 +311,6 @@ def compute_call_result(contract_valuation, call_requirement, market_simulation, def evaluate_dsl_expr(dsl_expr, first_commodity_name, simulation_id, interest_rate, present_time, simulated_value_dict, perturbation_dependencies, dependency_results, path_count, perturbation_factor): - evaluation_kwds = { 'simulated_value_dict': simulated_value_dict, 'simulation_id': simulation_id, @@ -432,17 +357,6 @@ def evaluate_dsl_expr(dsl_expr, first_commodity_name, simulation_id, interest_ra return result_value, perturbed_values -dsl_expr_pool = None - - -def get_compute_pool(): - # return None - global dsl_expr_pool - if dsl_expr_pool is None: - dsl_expr_pool = Pool(processes=4) - return dsl_expr_pool - - def get_dependency_results(contract_valuation_id, call_id, dependencies_repo, result_repo): assert isinstance(result_repo, (CallResultRepository, dict)), result_repo dependency_results = {} diff --git a/quantdsl/domain/services/dependency_graphs.py b/quantdsl/domain/services/dependency_graphs.py index 26c6696..e913a2d 100644 --- a/quantdsl/domain/services/dependency_graphs.py +++ b/quantdsl/domain/services/dependency_graphs.py @@ -224,7 +224,7 @@ def get(self): class PythonPendingCallQueue(PendingCallQueue): def __init__(self): - self.queue = queue.Queue() + self.queue = queue.LifoQueue() def put_pending_call(self, pending_call): self.queue.put(pending_call) diff --git a/quantdsl/domain/services/uuids.py b/quantdsl/domain/services/uuids.py index 4c91ebd..f9e56a8 100644 --- a/quantdsl/domain/services/uuids.py +++ b/quantdsl/domain/services/uuids.py @@ -1,4 +1,3 @@ - def create_uuid4(): import uuid return uuid.uuid4().hex diff --git a/quantdsl/tests/test_application.py b/quantdsl/tests/test_application.py index 1183f16..899d8b1 100644 --- a/quantdsl/tests/test_application.py +++ b/quantdsl/tests/test_application.py @@ -1,6 +1,5 @@ import datetime import unittest - from abc import ABCMeta from time import sleep @@ -9,13 +8,11 @@ from six import with_metaclass from quantdsl.application.with_pythonobjects import QuantDslApplicationWithPythonObjects -from quantdsl.domain.model.call_result import make_call_result_id, CallResult -from quantdsl.domain.model.contract_valuation import create_contract_valuation_id, ContractValuation +from quantdsl.domain.model.call_result import CallResult +from quantdsl.domain.model.contract_valuation import ContractValuation from quantdsl.domain.model.market_calibration import MarketCalibration from quantdsl.domain.model.market_simulation import MarketSimulation from quantdsl.domain.model.simulated_price import make_simulated_price_id -# from quantdsl.domain.services.call_links import regenerate_execution_order -# from quantdsl.semantics import Market from quantdsl.services import DEFAULT_PRICE_PROCESS_NAME @@ -23,7 +20,7 @@ class ApplicationTestCaseMixin(with_metaclass(ABCMeta)): skip_assert_event_handers_empty = False NUMBER_DAYS = 5 NUMBER_MARKETS = 2 - NUMBER_WORKERS = 30 + NUMBER_WORKERS = 20 PATH_COUNT = 2000 def setUp(self): @@ -40,14 +37,13 @@ def tearDown(self): self.app.close() if not self.skip_assert_event_handers_empty: assert_event_handlers_empty() - # super(ContractValuationTestCase, self).tearDown() + # super(ContractValuationTestCase, self).tearDown() def setup_application(self): self.app = QuantDslApplicationWithPythonObjects() class TestCase(ApplicationTestCaseMixin, unittest.TestCase): - def setUp(self): super(TestCase, self).setUp() @@ -56,7 +52,6 @@ def tearDown(self): class ContractValuationTestCase(ApplicationTestCaseMixin): - price_process_name = DEFAULT_PRICE_PROCESS_NAME calibration_params = { '#1-LAST-PRICE': 10, @@ -138,7 +133,9 @@ def assert_contract_value(self, specification, expected_value, expected_deltas=N market_calibration = self.app.market_calibration_repo[market_simulation.market_calibration_id] assert isinstance(market_calibration, MarketCalibration) commodity_name = perturbation.split('-')[0] - simulated_price_id = make_simulated_price_id(market_simulation.id, commodity_name, market_simulation.observation_date, market_simulation.observation_date) + simulated_price_id = make_simulated_price_id(market_simulation.id, commodity_name, + market_simulation.observation_date, + market_simulation.observation_date) simulated_price = self.app.simulated_price_repo[simulated_price_id] dy = perturbed_value - main_result.result_value @@ -180,7 +177,6 @@ def sleep(self, interval): class ExpressionTests(ContractValuationTestCase): - def test_generate_valuation_addition(self): self.assert_contract_value("""1 + 2""", 3) self.assert_contract_value("""2 + 4""", 6) @@ -361,7 +357,6 @@ def test_brownian_increments(self): class FunctionTests(ContractValuationTestCase): - def test_functional_fibonacci_numbers(self): fib_tmpl = """ def fib(n): return fib(n-1) + fib(n-2) if n > 1 else n @@ -426,7 +421,7 @@ def Option(date, strike, underlying, alternative): American(Date('%(starts)s'), Date('%(ends)s'), %(strike)s, Lift('%(underlying)s', Market('%(underlying)s'))) """ self.assert_contract_value(american_option_tmpl % { - 'starts':'2011-01-02', + 'starts': '2011-01-02', 'ends': '2011-01-04', 'strike': 9, 'underlying': '#1' @@ -449,13 +444,13 @@ def Swing(start_date, end_date, underlying, quantity): class LongerTests(ContractValuationTestCase): - def test_value_swing_option(self): specification = """ def Swing(start_date, end_date, underlying, quantity): if (quantity != 0) and (start_date < end_date): return Choice( - Swing(start_date + TimeDelta('1d'), end_date, underlying, quantity-1) + Fixing(start_date, Market(underlying)), + Swing(start_date + TimeDelta('1d'), end_date, underlying, quantity-1) + Fixing(start_date, + Market(underlying)), Swing(start_date + TimeDelta('1d'), end_date, underlying, quantity) ) else: @@ -501,7 +496,6 @@ def ProfitFromRunning(start_date, underlying, time_since_off): class SpecialTests(ContractValuationTestCase): - def test_simple_expression_with_market(self): dsl = "Market('NBP') + 2 * Market('TTF')" self.assert_contract_value(dsl, 32, {('NBP', 2011, 1): 1, ('TTF', 2011, 1): 2}, expected_call_count=1) @@ -577,13 +571,13 @@ def Hold(f, start_date, end_date, underlying, quantity): class ExperimentalTests(ContractValuationTestCase): - def test_value_swing_option_with_fixing_on_market(self): specification = """ def Swing(start_date, end_date, underlying, quantity): if (quantity != 0) and (start_date < end_date): return Choice( - Swing(start_date + TimeDelta('1d'), end_date, underlying, quantity-1) + Fixing(start_date, Market(underlying)), + Swing(start_date + TimeDelta('1d'), end_date, underlying, quantity-1) + Fixing(start_date, + Market(underlying)), Swing(start_date + TimeDelta('1d'), end_date, underlying, quantity) ) else: @@ -608,7 +602,6 @@ def Swing(start_date, end_date, underlying, quantity): """ self.assert_contract_value(specification, 30.0000, expected_call_count=15) - def test_value_swing_option_with_settlements_and_fixings_on_choice(self): specification = """ def Swing(start_date, end_date, underlying, quantity): @@ -766,7 +759,6 @@ def Inject(start, end, commodity_name, quantity, limit, step, vol): class SingleTests(ContractValuationTestCase): - def test_value_swing_option_with_forward_markets(self): specification = """ def Swing(start_date, end_date, quantity): @@ -798,4 +790,3 @@ class ContractValuationTests( FunctionTests, LongerTests ): pass - diff --git a/quantdsl/tests/test_multiprocessing.py b/quantdsl/tests/test_multiprocessing.py deleted file mode 100644 index 402e56c..0000000 --- a/quantdsl/tests/test_multiprocessing.py +++ /dev/null @@ -1,74 +0,0 @@ -import unittest - -from multiprocessing import Array -from multiprocessing.pool import Pool - -import os - -import scipy - -_d1 = None -_d2 = None - -def init_process(*args): - global _d1, _d2 - _d1 = args[0] - _d2 = args[1] - - -def get_value(id): - return numpy_from_array(v=_d1[id]) - - -def set_value(id, value): - _d2[id][:] = value - - -def numpy_from_array(v): - # return scipy.frombuffer(v.get_obj()) - return scipy.frombuffer(v.get_obj()) - - -def calc_result(id): - value = get_value(id) - assert value is not None - result = value - for i in range(1000): - result = result * value - set_value(id, result) - - -def echo(a): - # for i in range(10000): - # a = a * a - return a - -class TestMultiprocessing(unittest.TestCase): - - def test_pickle_numpy(self): - a = scipy.ones(200000) - pool = Pool(processes=1) - results = pool.map(echo, [a] * 1000) - for result in results: - assert result.all() == scipy.ones(10000).all(), result - - def test_sharedmem(self): - path_count = 2000 - d1 = {} - d2 = {} - node_count = 100 - for i in range(node_count): - np1_vi = scipy.ones(path_count) - v1 = Array('d', np1_vi) - np2_vi = scipy.zeros(path_count) - v2 = Array('d', np2_vi) - d1[i] = v1 - d2[i] = v2 - p = Pool(processes=4, initializer=init_process, initargs=(d1, d2)) - results = [] - for i in range(node_count): - results.append(p.apply_async(func=calc_result, args=(i,))) - [r.get() for r in results] - np_array = numpy_from_array(d2[0]) - np_array_mean = np_array.mean() - self.assertEqual(np_array_mean, 1) From d89945e76bc09b9a78459c55a64ff792994a973f Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 21:06:04 +0100 Subject: [PATCH 16/52] Removed more unused stuff. --- quantdsl/application/base.py | 8 +- quantdsl/application/call_result_policy.py | 27 ++++--- .../domain/services/contract_valuations.py | 75 ++++--------------- .../infrastructure/evaluation_subscriber.py | 13 ++-- quantdsl/tests/test_evaluation_subscriber.py | 1 - 5 files changed, 39 insertions(+), 85 deletions(-) diff --git a/quantdsl/application/base.py b/quantdsl/application/base.py index b996fa3..33e27a7 100644 --- a/quantdsl/application/base.py +++ b/quantdsl/application/base.py @@ -50,7 +50,7 @@ class QuantDslApplication(EventSourcingApplication): Evaluate contract given call dependency graph and market simulation. """ - def __init__(self, call_evaluation_queue=None, result_counters=None, *args, **kwargs): + def __init__(self, call_evaluation_queue=None, *args, **kwargs): super(QuantDslApplication, self).__init__(*args, **kwargs) self.contract_specification_repo = ContractSpecificationRepo(event_store=self.event_store, use_cache=True) self.contract_valuation_repo = ContractValuationRepo(event_store=self.event_store, use_cache=True) @@ -69,7 +69,6 @@ def __init__(self, call_evaluation_queue=None, result_counters=None, *args, **kw # self.call_result_repo = CallResultRepo(event_store=self.event_store, use_cache=True) self.call_result_repo = {} self.call_evaluation_queue = call_evaluation_queue - self.result_counters = result_counters self.simulation_subscriber = SimulationSubscriber( market_calibration_repo=self.market_calibration_repo, @@ -92,7 +91,6 @@ def __init__(self, call_evaluation_queue=None, result_counters=None, *args, **kw market_simulation_repo=self.market_simulation_repo, call_evaluation_queue=self.call_evaluation_queue, call_leafs_repo=self.call_leafs_repo, - result_counters=self.result_counters, call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, @@ -155,7 +153,7 @@ def start_contract_valuation(self, contract_specification_id, market_simulation_ assert isinstance(contract_specification_id, six.string_types), contract_specification_id return start_contract_valuation(contract_specification_id, market_simulation_id) - def loop_on_evaluation_queue(self, compute_pool=None): + def loop_on_evaluation_queue(self): loop_on_evaluation_queue( call_evaluation_queue=self.call_evaluation_queue, contract_valuation_repo=self.contract_valuation_repo, @@ -167,7 +165,6 @@ def loop_on_evaluation_queue(self, compute_pool=None): call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, - compute_pool=compute_pool, ) def evaluate_call_and_queue_next_calls(self, contract_valuation_id, contract_specification_id, call_id): @@ -181,7 +178,6 @@ def evaluate_call_and_queue_next_calls(self, contract_valuation_id, contract_spe call_dependencies_repo=self.call_dependencies_repo, call_result_repo=self.call_result_repo, simulated_price_repo=self.simulated_price_repo, - call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_requirements_repo=self.simulated_price_requirements_repo, ) diff --git a/quantdsl/application/call_result_policy.py b/quantdsl/application/call_result_policy.py index 93d0472..1475d8c 100644 --- a/quantdsl/application/call_result_policy.py +++ b/quantdsl/application/call_result_policy.py @@ -13,8 +13,8 @@ def __init__(self, call_result_repo, call_evaluation_queue=None): self.result = {} self.dependents = {} self.dependencies = {} - self.usages = {} - self.result_counters = {} + self.outstanding_dependents = {} + self.outstanding_dependencies = {} subscribe(self.is_call_dependencies_created, self.cache_dependencies) subscribe(self.is_call_dependents_created, self.cache_dependents) @@ -43,13 +43,20 @@ def cache_dependencies(self, event): assert isinstance(event, CallDependencies.Created) call_result_id = event.entity_id self.dependencies[call_result_id] = event.dependencies[:] - self.result_counters[call_result_id] = [None] * (len(event.dependencies) - 1) + + # Count one outstanding dependency for each dependency of the call. + # - minus 1, so that pop() raises at the right time, see below + if self.call_evaluation_queue: + self.outstanding_dependencies[call_result_id] = [None] * (len(event.dependencies) - 1) def cache_dependents(self, event): assert isinstance(event, CallDependents.Created) call_result_id = event.entity_id self.dependents[call_result_id] = event.dependents[:] - self.usages[call_result_id] = [None] * (len(event.dependents) - 1) + + # Count one outstanding dependent for each dependent of the call. + # - minus 1, so that pop() raises at the right time, see below + self.outstanding_dependents[call_result_id] = [None] * (len(event.dependents) - 1) def cache_result(self, event): assert isinstance(event, CallResult.Created) @@ -61,22 +68,22 @@ def cache_result(self, event): if isinstance(self.call_result_repo, dict): self.call_result_repo[this_result_id] = call_result - # Remove one usage for each dependency of this result. + # Decrement outstanding dependents for each dependency of this result. for dependency_id in self.dependencies.get(event.call_id, ()): try: - self.usages[dependency_id].pop() + self.outstanding_dependents[dependency_id].pop() except IndexError: - # Discard the result when it has been fully used. + # Discard the result when there are no longer any outstanding dependents. dependent_result_id = make_call_result_id(event.contract_valuation_id, dependency_id) self.result[dependent_result_id].discard() - # Remove one result for each dependent of this result. + # Remove one outstanding dependency for each dependent of this result. if self.call_evaluation_queue: for dependent_id in self.dependents.get(event.call_id, ()): try: - self.result_counters[dependent_id].pop() + self.outstanding_dependencies[dependent_id].pop() except IndexError: - # Queue the call if a dependent has all its results. + # Queue the call if there are no more outstanding results. job = (event.contract_specification_id, event.contract_valuation_id, dependent_id) self.call_evaluation_queue.put(job) diff --git a/quantdsl/domain/services/contract_valuations.py b/quantdsl/domain/services/contract_valuations.py index d829d96..320bc31 100644 --- a/quantdsl/domain/services/contract_valuations.py +++ b/quantdsl/domain/services/contract_valuations.py @@ -3,7 +3,6 @@ from eventsourcing.domain.model.events import publish from quantdsl.domain.model.call_dependencies import CallDependencies -from quantdsl.domain.model.call_dependents import CallDependents from quantdsl.domain.model.call_requirement import CallRequirement from quantdsl.domain.model.call_result import CallResult, CallResultRepository, ResultValueComputed, \ make_call_result_id, register_call_result @@ -18,8 +17,7 @@ def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, call_evaluation_queue, call_leafs_repo, call_link_repo, call_requirement_repo, call_result_repo, contract_valuation_repo, - market_simulation_repo, simulated_price_repo, result_counters, - call_dependents_repo, perturbation_dependencies_repo, + market_simulation_repo, simulated_price_repo, perturbation_dependencies_repo, simulated_price_dependencies_repo): if not call_evaluation_queue: evaluate_contract_in_series( @@ -29,7 +27,6 @@ def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, c simulated_price_repo=simulated_price_repo, call_requirement_repo=call_requirement_repo, call_dependencies_repo=call_dependencies_repo, - call_dependents_repo=call_dependents_repo, call_link_repo=call_link_repo, call_result_repo=call_result_repo, perturbation_dependencies_repo=perturbation_dependencies_repo, @@ -41,19 +38,13 @@ def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, c contract_valuation_repo=contract_valuation_repo, call_leafs_repo=call_leafs_repo, call_evaluation_queue=call_evaluation_queue, - call_link_repo=call_link_repo, - result_counters=result_counters, - call_dependencies_repo=call_dependencies_repo, - call_dependents_repo=call_dependents_repo, - perturbation_dependencies_repo=perturbation_dependencies_repo, - simulated_price_dependencies_repo=simulated_price_dependencies_repo, ) def evaluate_contract_in_series(contract_valuation_id, contract_valuation_repo, market_simulation_repo, simulated_price_repo, call_requirement_repo, call_dependencies_repo, - call_dependents_repo, call_link_repo, - call_result_repo, perturbation_dependencies_repo, simulated_price_dependencies_repo): + call_link_repo, call_result_repo, perturbation_dependencies_repo, + simulated_price_dependencies_repo): """ Computes value of contract by following the series execution order of its call dependency graph in directly evaluating each call in turn until the whole graph has been evaluated. @@ -89,7 +80,6 @@ def evaluate_contract_in_series(contract_valuation_id, contract_valuation_repo, perturbation_dependencies=perturbation_dependencies, call_dependencies_repo=call_dependencies_repo, call_result_repo=call_result_repo, - perturbation_dependencies_repo=perturbation_dependencies_repo, simulated_price_repo=simulated_price_repo, simulation_requirements=simulation_requirements ) @@ -104,10 +94,8 @@ def evaluate_contract_in_series(contract_valuation_id, contract_valuation_repo, ) -def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo, call_leafs_repo, call_link_repo, - call_evaluation_queue, result_counters, call_dependencies_repo, - call_dependents_repo, perturbation_dependencies_repo, - simulated_price_dependencies_repo): +def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo, call_leafs_repo, + call_evaluation_queue): """ Computes value of contract by putting the dependency graph leaves on an evaluation queue and expecting there is at least one worker loop evaluating the queued calls and putting satisfied dependents on the queue. @@ -118,18 +106,6 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo contract_specification_id = contract_valuation.contract_specification_id - if result_counters is not None: - for call_id in regenerate_execution_order(contract_specification_id, call_link_repo): - call_dependencies = call_dependencies_repo[call_id] - call_dependents = call_dependents_repo[call_id] - assert isinstance(call_dependencies, CallDependencies) - assert isinstance(call_dependents, CallDependents) - count_dependencies = len(call_dependencies.dependencies) - # Crude attempt to count down using atomic operations, so we get an exception when we can't pop off the - # last one. - call_result_id = make_call_result_id(contract_valuation_id, call_id) - result_counters[call_result_id] = [None] * (count_dependencies - 1) - call_leafs = call_leafs_repo[contract_specification_id] # assert isinstance(call_leafs, CallLeafs) @@ -139,8 +115,7 @@ def evaluate_contract_in_parallel(contract_valuation_id, contract_valuation_repo def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, call_requirement_repo, market_simulation_repo, call_dependencies_repo, call_result_repo, simulated_price_repo, - call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo, - compute_pool=None): + call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo): while True: item = call_evaluation_queue.get() try: @@ -156,11 +131,8 @@ def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, cal call_dependencies_repo=call_dependencies_repo, call_result_repo=call_result_repo, simulated_price_repo=simulated_price_repo, - call_dependents_repo=call_dependents_repo, perturbation_dependencies_repo=perturbation_dependencies_repo, - simulated_price_requirements_repo=simulated_price_requirements_repo, - compute_pool=compute_pool, - ) + simulated_price_requirements_repo=simulated_price_requirements_repo) finally: call_evaluation_queue.task_done() @@ -168,9 +140,7 @@ def loop_on_evaluation_queue(call_evaluation_queue, contract_valuation_repo, cal def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specification_id, call_id, contract_valuation_repo, call_requirement_repo, market_simulation_repo, call_dependencies_repo, call_result_repo, simulated_price_repo, - call_dependents_repo, perturbation_dependencies_repo, - simulated_price_requirements_repo, - compute_pool=None): + perturbation_dependencies_repo, simulated_price_requirements_repo): # Get the contract valuation. contract_valuation = contract_valuation_repo[contract_valuation_id] assert isinstance(contract_valuation, ContractValuation) @@ -197,9 +167,7 @@ def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specifica call_dependencies_repo=call_dependencies_repo, call_result_repo=call_result_repo, simulated_price_repo=simulated_price_repo, - perturbation_dependencies_repo=perturbation_dependencies_repo, simulation_requirements=simulation_requirements, - compute_pool=compute_pool, ) # Register the call result. @@ -240,9 +208,7 @@ def find_dependents_ready_to_be_evaluated(contract_valuation_id, call_id, call_d def compute_call_result(contract_valuation, call_requirement, market_simulation, perturbation_dependencies, - call_dependencies_repo, call_result_repo, simulated_price_repo, - perturbation_dependencies_repo, simulation_requirements, - compute_pool=None): + call_dependencies_repo, call_result_repo, simulated_price_repo, simulation_requirements): """ Parses, compiles and evaluates a call requirement. """ @@ -287,23 +253,12 @@ def compute_call_result(contract_valuation, call_requirement, market_simulation, result_repo=call_result_repo, ) - # Compute the call result. - if compute_pool is None: - result_value, perturbed_values = evaluate_dsl_expr(dsl_expr, first_commodity_name, market_simulation.id, - market_simulation.interest_rate, present_time, - simulated_value_dict, - perturbation_dependencies.dependencies, - dependency_results, market_simulation.path_count, - market_simulation.perturbation_factor) - else: - assert isinstance(compute_pool, Pool) - async_result = compute_pool.apply_async( - evaluate_dsl_expr, - args=(dsl_expr, first_commodity_name, market_simulation.id, market_simulation.interest_rate, - present_time, simulated_value_dict, perturbation_dependencies.dependencies, dependency_results, - market_simulation.path_count, market_simulation.perturbation_factor), - ) - result_value, perturbed_values = async_result.get() + result_value, perturbed_values = evaluate_dsl_expr(dsl_expr, first_commodity_name, market_simulation.id, + market_simulation.interest_rate, present_time, + simulated_value_dict, + perturbation_dependencies.dependencies, + dependency_results, market_simulation.path_count, + market_simulation.perturbation_factor) # Return the result. return result_value, perturbed_values diff --git a/quantdsl/infrastructure/evaluation_subscriber.py b/quantdsl/infrastructure/evaluation_subscriber.py index 9946d18..af27e09 100644 --- a/quantdsl/infrastructure/evaluation_subscriber.py +++ b/quantdsl/infrastructure/evaluation_subscriber.py @@ -18,7 +18,7 @@ class EvaluationSubscriber(object): def __init__(self, contract_valuation_repo, call_link_repo, call_dependencies_repo, call_requirement_repo, call_result_repo, simulated_price_repo, market_simulation_repo, call_leafs_repo, - call_evaluation_queue, result_counters, call_dependents_repo, + call_evaluation_queue, call_dependents_repo, perturbation_dependencies_repo, simulated_price_requirements_repo): assert isinstance(contract_valuation_repo, ContractValuationRepository), contract_valuation_repo assert isinstance(call_link_repo, CallLinkRepository), call_link_repo @@ -30,7 +30,6 @@ def __init__(self, contract_valuation_repo, call_link_repo, call_dependencies_re assert isinstance(call_dependents_repo, CallDependentsRepository), call_dependents_repo assert isinstance(perturbation_dependencies_repo, PerturbationDependenciesRepo), perturbation_dependencies_repo assert isinstance(simulated_price_requirements_repo, SimulatedPriceRequirementsRepo), simulated_price_requirements_repo - # assert isinstance(result_counters, dict), result_counters self.contract_valuation_repo = contract_valuation_repo self.call_link_repo = call_link_repo self.call_dependencies_repo = call_dependencies_repo @@ -40,16 +39,16 @@ def __init__(self, contract_valuation_repo, call_link_repo, call_dependencies_re self.market_simulation_repo = market_simulation_repo self.call_leafs_repo = call_leafs_repo self.call_evaluation_queue = call_evaluation_queue - self.result_counters = result_counters self.call_dependents_repo = call_dependents_repo self.perturbation_dependencies_repo = perturbation_dependencies_repo self.simulated_price_dependencies_repo = simulated_price_requirements_repo - subscribe(self.contract_valuation_created, self.generate_contract_valuation) + subscribe(self.is_contract_valuation_created, self.generate_contract_valuation) def close(self): - unsubscribe(self.contract_valuation_created, self.generate_contract_valuation) + unsubscribe(self.is_contract_valuation_created, self.generate_contract_valuation) - def contract_valuation_created(self, event): + @staticmethod + def is_contract_valuation_created(event): return isinstance(event, ContractValuation.Created) def generate_contract_valuation(self, event): @@ -64,8 +63,6 @@ def generate_contract_valuation(self, event): contract_valuation_repo=self.contract_valuation_repo, market_simulation_repo=self.market_simulation_repo, simulated_price_repo=self.simulated_price_repo, - result_counters=self.result_counters, - call_dependents_repo=self.call_dependents_repo, perturbation_dependencies_repo=self.perturbation_dependencies_repo, simulated_price_dependencies_repo=self.simulated_price_dependencies_repo, ) diff --git a/quantdsl/tests/test_evaluation_subscriber.py b/quantdsl/tests/test_evaluation_subscriber.py index 7c2e942..e433d66 100644 --- a/quantdsl/tests/test_evaluation_subscriber.py +++ b/quantdsl/tests/test_evaluation_subscriber.py @@ -41,7 +41,6 @@ def setUp(self): market_simulation_repo=market_simulation_repo, call_leafs_repo=call_leafs_repo, call_evaluation_queue=None, - result_counters=None, call_dependents_repo=call_dependents_repo, perturbation_dependencies_repo=perturbation_dependencies_repo, simulated_price_requirements_repo=simulated_price_requirements_repo From 3a141cb974c9332caec2e373e34f4db7ce342072 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 21:30:01 +0100 Subject: [PATCH 17/52] Added py3.5 to list of travis targets. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ab32f07..8932f81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "2.7" - "3.3" - "3.4" + - "3.5" before_install: - travis_retry wget http://repo.continuum.io/miniconda/Miniconda-3.8.3-Linux-x86_64.sh -O miniconda.sh From 575b098ac00cbb5c98dffdb7c0921b2be8db55c0 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 21:36:34 +0100 Subject: [PATCH 18/52] Added py3.5 to list of trove classifiers. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4147b37..f4eabb5 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Office/Business :: Financial', 'Topic :: Office/Business :: Financial :: Investment', 'Topic :: Office/Business :: Financial :: Spreadsheet', From 7066ed66421df05ae9e0f408f145ebe68205830c Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 21:44:59 +0100 Subject: [PATCH 19/52] Removed unused function. --- .../domain/services/contract_valuations.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/quantdsl/domain/services/contract_valuations.py b/quantdsl/domain/services/contract_valuations.py index 320bc31..d8603db 100644 --- a/quantdsl/domain/services/contract_valuations.py +++ b/quantdsl/domain/services/contract_valuations.py @@ -180,33 +180,6 @@ def evaluate_call_and_queue_next_calls(contract_valuation_id, contract_specifica ) -def find_dependents_ready_to_be_evaluated(contract_valuation_id, call_id, call_dependents_repo, call_dependencies_repo, - call_result_repo, result_counters): - # assert isinstance(contract_valuation_id, six.string_types), contract_valuation_id - # assert isinstance(call_id, six.string_types), call_id - # assert isinstance(call_dependents_repo, CallDependentsRepository) - # assert isinstance(call_dependencies_repo, CallDependenciesRepository) - # assert isinstance(call_result_repo, CallResultRepository) - assert result_counters is not None - - # Get dependents (if any exist). - try: - call_dependents = call_dependents_repo[call_id] - except KeyError: - # Don't worry if there are none. - pass - else: - # Check if any dependents are ready to be evaluated. - ready = [] - for dependent_id in call_dependents.dependents: - dependent_result_id = make_call_result_id(contract_valuation_id, dependent_id) - try: - result_counters[dependent_result_id].pop() # Pop one off the array (atomic decrement). - except (KeyError, IndexError): - ready.append(dependent_id) - return ready - - def compute_call_result(contract_valuation, call_requirement, market_simulation, perturbation_dependencies, call_dependencies_repo, call_result_repo, simulated_price_repo, simulation_requirements): """ From 2c4649e700bc143e68d2310b7c1151921c475db6 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 22:29:52 +0100 Subject: [PATCH 20/52] Removed more unused stuff. --- quantdsl/domain/model/call_result.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/quantdsl/domain/model/call_result.py b/quantdsl/domain/model/call_result.py index 66ae142..9c9667b 100644 --- a/quantdsl/domain/model/call_result.py +++ b/quantdsl/domain/model/call_result.py @@ -17,10 +17,6 @@ def contract_valuation_id(self): def contract_specification_id(self): return self.__dict__['contract_specification_id'] - @property - def result_value(self): - return self.__dict__['result_value'] - class Discarded(EventSourcedEntity.Discarded): pass @@ -33,10 +29,6 @@ def __init__(self, result_value, perturbed_values, contract_valuation_id, call_i self._call_id = call_id self._contract_specification_id = contract_specification_id - @property - def call_id(self): - return self._call_id - @property def result_value(self): return self._result_value @@ -45,21 +37,6 @@ def result_value(self): def perturbed_values(self): return self._perturbed_values - @property - def contract_valuation_id(self): - return self._contract_valuation_id - - @property - def contract_specification_id(self): - return self._contract_valuation_id - - @property - def scalar_result_value(self): - result_value = self._result_value - if isinstance(result_value, scipy.ndarray): - result_value = result_value.mean() - return result_value - def register_call_result(call_id, result_value, perturbed_values, contract_valuation_id, contract_specification_id): call_result_id = make_call_result_id(contract_valuation_id, call_id) From c3ba251cbdf55ae64bf7cb6bcc2e350aa0867cfb Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 22:48:01 +0100 Subject: [PATCH 21/52] Removed more unused stuff. --- quantdsl/syntax.py | 48 +++++------------------------------ quantdsl/tests/test_parser.py | 11 ++++++++ 2 files changed, 18 insertions(+), 41 deletions(-) diff --git a/quantdsl/syntax.py b/quantdsl/syntax.py index e514f42..c926378 100644 --- a/quantdsl/syntax.py +++ b/quantdsl/syntax.py @@ -66,17 +66,6 @@ def visitModule(self, node): # Namespace for function defs in module. module_namespace = DslNamespace() - def inline(body): - flat = [] - assert isinstance(body, list), body - for obj in body: - if isinstance(obj, list): - for _obj in inline(obj): - flat.append(_obj) - else: - flat.append(obj) - return flat - for n in node.body: dsl_object = self.visitAstNode(n) @@ -89,7 +78,7 @@ def inline(body): # Include imported things. if isinstance(dsl_object, list): - for _dsl_object in inline(dsl_object): + for _dsl_object in dsl_object: if isinstance(_dsl_object, FunctionDef): module_namespace[_dsl_object.name] = _dsl_object else: @@ -97,23 +86,6 @@ def inline(body): return self.dsl_classes['Module'](body, module_namespace, node=node) - # def visitImport(self, node): - # """ - # Visitor method for ast.Import nodes. - # - # Returns the result of visiting the expression held by the return statement. - # """ - # assert isinstance(node, ast.Import) - # nodes = [] - # for imported in node.names: - # name = imported.name - # if name == 'quantdsl.semantics': - # continue - # # spec = importlib.util.find_spec() - # node = self.import_python_module(name, node, nodes) - # - # return nodes - def visitImportFrom(self, node): """ Visitor method for ast.ImportFrom nodes. @@ -156,10 +128,8 @@ def visitExpr(self, node): Returns the result of visiting the contents of the expression node. """ # assert isinstance(node, ast.Expr) - if isinstance(node.value, ast.AST): - return self.visitAstNode(node.value) - else: - raise DslSyntaxError + assert isinstance(node.value, ast.AST), type(node.value) + return self.visitAstNode(node.value) def visitNum(self, node): """ @@ -227,14 +197,10 @@ def visitBoolOp(self, node): ast.And: self.dsl_classes['And'], ast.Or: self.dsl_classes['Or'], } - try: - dsl_class = type_map[type(node.op)] - except KeyError: - raise DslSyntaxError("Unsupported boolean operator token: %s" % node.op) - else: - values = [self.visitAstNode(v) for v in node.values] - args = [values] - return dsl_class(node=node, *args) + dsl_class = type_map[type(node.op)] + values = [self.visitAstNode(v) for v in node.values] + args = [values] + return dsl_class(node=node, *args) def visitName(self, node): """ diff --git a/quantdsl/tests/test_parser.py b/quantdsl/tests/test_parser.py index a241cbd..cd4ba9c 100644 --- a/quantdsl/tests/test_parser.py +++ b/quantdsl/tests/test_parser.py @@ -25,10 +25,21 @@ def setUp(self): self.p = DslParser() def test_empty_string(self): + # Empty string can be parsed as a module, but not compiled into an expression, or evaluated. self.assertTrue(isinstance(dsl_parse(""), Module)) self.assertRaises(DslSyntaxError, dsl_compile, "") self.assertRaises(DslSyntaxError, dsl_eval, "") + def test_invalid_source(self): + # Check a non-valid Python string raises an error. + self.assertRaises(DslSyntaxError, dsl_parse, "1 +") + # Check a non-string raises an error. + self.assertRaises(DslSyntaxError, dsl_compile, 1) + + def test_unsupported_elements(self): + # Check an unsupported element isn't parsed. + self.assertRaises(DslSyntaxError, dsl_parse, "pass") + def assertDslExprTypeValue(self, dsl_source, expectedDslType, expectedDslValue, **compile_kwds): # Assumes dsl_source is just one statement. dsl_module = dsl_parse(dsl_source) From 25c1836e64719b5f51d28d02264b937fe652f84a Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 23:05:37 +0100 Subject: [PATCH 22/52] Removed more unused stuff. --- quantdsl/domain/model/simulated_price_requirements.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/quantdsl/domain/model/simulated_price_requirements.py b/quantdsl/domain/model/simulated_price_requirements.py index b5f24f2..9262c7e 100644 --- a/quantdsl/domain/model/simulated_price_requirements.py +++ b/quantdsl/domain/model/simulated_price_requirements.py @@ -17,9 +17,6 @@ def __init__(self, requirements, **kwargs): super(SimulatedPriceRequirements, self).__init__(**kwargs) self._requirements = requirements - def __getitem__(self, item): - return self._requirements.__getitem__(item) - @property def requirements(self): return self._requirements From 8a37249915d4d2d9137c593f34409583aea32bba Mon Sep 17 00:00:00 2001 From: John Bywater Date: Mon, 18 Sep 2017 23:06:57 +0100 Subject: [PATCH 23/52] Removed more unused stuff. --- quantdsl/domain/model/perturbation_dependencies.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/quantdsl/domain/model/perturbation_dependencies.py b/quantdsl/domain/model/perturbation_dependencies.py index d77f9cb..38bbcc5 100644 --- a/quantdsl/domain/model/perturbation_dependencies.py +++ b/quantdsl/domain/model/perturbation_dependencies.py @@ -17,9 +17,6 @@ def __init__(self, dependencies, **kwargs): super(PerturbationDependencies, self).__init__(**kwargs) self._dependencies = dependencies - def __getitem__(self, item): - return self._dependencies.__getitem__(item) - @property def dependencies(self): return self._dependencies From d021819133112e573bed150b31db2e072c6b1b6d Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 03:48:24 +0100 Subject: [PATCH 24/52] Changed to use a Python dict for the simulated price repo. --- quantdsl/application/base.py | 4 +++- quantdsl/application/persistence_policy.py | 8 ++++++-- quantdsl/application/with_multithreading.py | 4 ++-- .../domain/services/contract_valuations.py | 20 +++++++++---------- quantdsl/domain/services/simulated_prices.py | 2 +- .../infrastructure/evaluation_subscriber.py | 2 +- .../infrastructure/simulation_subscriber.py | 8 ++++++-- quantdsl/semantics.py | 2 ++ ...n_with_multithreading_and_pythonobjects.py | 2 +- quantdsl/tests/test_domain_services.py | 3 ++- quantdsl/tests/test_eventsourced_repos.py | 2 ++ quantdsl/tests/test_simulation_subscriber.py | 8 +++++++- 12 files changed, 42 insertions(+), 23 deletions(-) diff --git a/quantdsl/application/base.py b/quantdsl/application/base.py index 33e27a7..5620d1f 100644 --- a/quantdsl/application/base.py +++ b/quantdsl/application/base.py @@ -60,7 +60,8 @@ def __init__(self, call_evaluation_queue=None, *args, **kwargs): use_cache=True) self.simulated_price_requirements_repo = SimulatedPriceRequirementsRepo(event_store=self.event_store, use_cache=True) - self.simulated_price_repo = SimulatedPriceRepo(event_store=self.event_store, use_cache=True) + # self.simulated_price_repo = SimulatedPriceRepo(event_store=self.event_store, use_cache=True) + self.simulated_price_repo = {} self.call_requirement_repo = CallRequirementRepo(event_store=self.event_store, use_cache=True) self.call_dependencies_repo = CallDependenciesRepo(event_store=self.event_store, use_cache=True) self.call_dependents_repo = CallDependentsRepo(event_store=self.event_store, use_cache=True) @@ -73,6 +74,7 @@ def __init__(self, call_evaluation_queue=None, *args, **kwargs): self.simulation_subscriber = SimulationSubscriber( market_calibration_repo=self.market_calibration_repo, market_simulation_repo=self.market_simulation_repo, + simulated_price_repo=self.simulated_price_repo ) self.dependency_graph_subscriber = DependencyGraphSubscriber( contract_specification_repo=self.contract_specification_repo, diff --git a/quantdsl/application/persistence_policy.py b/quantdsl/application/persistence_policy.py index 6be12fe..d304fff 100644 --- a/quantdsl/application/persistence_policy.py +++ b/quantdsl/application/persistence_policy.py @@ -1,11 +1,15 @@ from eventsourcing.infrastructure.persistence_subscriber import PersistenceSubscriber from quantdsl.domain.model.call_result import CallResult +from quantdsl.domain.model.simulated_price import SimulatedPrice class PersistencePolicy(PersistenceSubscriber): @staticmethod def is_domain_event(event): return PersistenceSubscriber.is_domain_event(event) and \ - not isinstance(event, (CallResult.Created, - CallResult.Discarded)) \ No newline at end of file + not isinstance(event, ( + CallResult.Created, + CallResult.Discarded, + SimulatedPrice.Created, + )) \ No newline at end of file diff --git a/quantdsl/application/with_multithreading.py b/quantdsl/application/with_multithreading.py index 257e8ec..b3b40bd 100644 --- a/quantdsl/application/with_multithreading.py +++ b/quantdsl/application/with_multithreading.py @@ -6,11 +6,11 @@ class QuantDslApplicationWithMultithreading(QuantDslApplication): - def __init__(self, num_workers, *args, **kwargs): + def __init__(self, num_threads=4, *args, **kwargs): super(QuantDslApplicationWithMultithreading, self).__init__(call_evaluation_queue=queue.LifoQueue(), *args, **kwargs) # Start evaluation worker threads. - for _ in range(num_workers): + for _ in range(num_threads): t = Thread(target=self.loop_on_evaluation_queue) t.setDaemon(True) t.start() diff --git a/quantdsl/domain/services/contract_valuations.py b/quantdsl/domain/services/contract_valuations.py index d8603db..c1c4bda 100644 --- a/quantdsl/domain/services/contract_valuations.py +++ b/quantdsl/domain/services/contract_valuations.py @@ -12,7 +12,7 @@ from quantdsl.domain.model.simulated_price import make_simulated_price_id from quantdsl.domain.services.call_links import regenerate_execution_order from quantdsl.domain.services.parser import dsl_parse -from quantdsl.semantics import DslNamespace +from quantdsl.semantics import DslNamespace, Module def generate_contract_valuation(contract_valuation_id, call_dependencies_repo, call_evaluation_queue, call_leafs_repo, @@ -186,21 +186,19 @@ def compute_call_result(contract_valuation, call_requirement, market_simulation, Parses, compiles and evaluates a call requirement. """ # assert isinstance(contract_valuation, ContractValuation), contract_valuation - assert isinstance(call_requirement, CallRequirement), call_requirement - assert isinstance(market_simulation, MarketSimulation), market_simulation + # assert isinstance(call_requirement, CallRequirement), call_requirement + # assert isinstance(market_simulation, MarketSimulation), market_simulation # assert isinstance(call_dependencies_repo, CallDependenciesRepository), call_dependencies_repo # assert isinstance(call_result_repo, CallResultRepository) # assert isinstance(simulated_price_dict, SimulatedPriceRepository) - # Todo: Put getting the dependency values in a separate thread, and perhaps make each call a separate thread. # Parse the DSL source into a DSL module object (composite tree of objects that provide the DSL semantics). - if call_requirement._dsl_expr is not None: - dsl_expr = call_requirement._dsl_expr - else: - dsl_module = dsl_parse(call_requirement.dsl_source) - dsl_expr = dsl_module.body[0] - - # assert isinstance(dsl_module, Module), "Parsed stubbed expr string is not a module: %s" % dsl_module + # if call_requirement._dsl_expr is not None: + dsl_expr = call_requirement._dsl_expr + # else: + # dsl_module = dsl_parse(call_requirement.dsl_source) + # assert isinstance(dsl_module, Module), "Parsed stubbed expr string is not a module: %s" % dsl_module + # dsl_expr = dsl_module.body[0] present_time = call_requirement.effective_present_time or market_simulation.observation_date diff --git a/quantdsl/domain/services/simulated_prices.py b/quantdsl/domain/services/simulated_prices.py index 0c73607..149fac6 100644 --- a/quantdsl/domain/services/simulated_prices.py +++ b/quantdsl/domain/services/simulated_prices.py @@ -15,7 +15,7 @@ def generate_simulated_prices(market_simulation, market_calibration): for commodity_name, fixing_date, delivery_date, price_value in simulate_future_prices(market_simulation, market_calibration): - register_simulated_price(market_simulation.id, commodity_name, fixing_date, delivery_date, price_value) + yield register_simulated_price(market_simulation.id, commodity_name, fixing_date, delivery_date, price_value) def simulate_future_prices(market_simulation, market_calibration): diff --git a/quantdsl/infrastructure/evaluation_subscriber.py b/quantdsl/infrastructure/evaluation_subscriber.py index af27e09..270fa1f 100644 --- a/quantdsl/infrastructure/evaluation_subscriber.py +++ b/quantdsl/infrastructure/evaluation_subscriber.py @@ -25,7 +25,7 @@ def __init__(self, contract_valuation_repo, call_link_repo, call_dependencies_re assert isinstance(call_dependencies_repo, CallDependenciesRepository), call_dependencies_repo assert isinstance(call_requirement_repo, CallRequirementRepository), call_requirement_repo assert isinstance(call_result_repo, (CallResultRepository, dict)), call_result_repo - assert isinstance(simulated_price_repo, SimulatedPriceRepository), simulated_price_repo + # assert isinstance(simulated_price_repo, SimulatedPriceRepository), simulated_price_repo assert isinstance(market_simulation_repo, MarketSimulationRepository), market_simulation_repo assert isinstance(call_dependents_repo, CallDependentsRepository), call_dependents_repo assert isinstance(perturbation_dependencies_repo, PerturbationDependenciesRepo), perturbation_dependencies_repo diff --git a/quantdsl/infrastructure/simulation_subscriber.py b/quantdsl/infrastructure/simulation_subscriber.py index b4070b8..b178aee 100644 --- a/quantdsl/infrastructure/simulation_subscriber.py +++ b/quantdsl/infrastructure/simulation_subscriber.py @@ -2,17 +2,20 @@ from quantdsl.domain.model.market_calibration import MarketCalibrationRepository from quantdsl.domain.model.market_simulation import MarketSimulation, MarketSimulationRepository +from quantdsl.domain.model.simulated_price import SimulatedPriceRepository from quantdsl.domain.services.simulated_prices import generate_simulated_prices class SimulationSubscriber(object): # When a market simulation is created, generate and register all the simulated prices. - def __init__(self, market_calibration_repo, market_simulation_repo): + def __init__(self, market_calibration_repo, market_simulation_repo, simulated_price_repo): assert isinstance(market_calibration_repo, MarketCalibrationRepository) assert isinstance(market_simulation_repo, MarketSimulationRepository) + # assert isinstance(simulated_price_repo, SimulatedPriceRepository) self.market_calibration_repo = market_calibration_repo self.market_simulation_repo = market_simulation_repo + self.simulated_price_repo = simulated_price_repo subscribe(self.market_simulation_created, self.generate_simulated_prices_for_market_simulation) def close(self): @@ -25,4 +28,5 @@ def generate_simulated_prices_for_market_simulation(self, event): assert isinstance(event, MarketSimulation.Created) market_simulation = self.market_simulation_repo[event.entity_id] market_calibration = self.market_calibration_repo[event.market_calibration_id] - generate_simulated_prices(market_simulation, market_calibration) + for simulated_price in generate_simulated_prices(market_simulation, market_calibration): + self.simulated_price_repo[simulated_price.id] = simulated_price diff --git a/quantdsl/semantics.py b/quantdsl/semantics.py index 0a0e636..ab8e65d 100644 --- a/quantdsl/semantics.py +++ b/quantdsl/semantics.py @@ -40,7 +40,9 @@ def __str__(self): """ return self.pprint() + # Todo: More tests that this round trip actually works. def pprint(self, indent=''): + """Returns Quant DSL source code for the DSL object.""" msg = self.__class__.__name__ + "(" lenArgs = len(self._args) if lenArgs > 1: diff --git a/quantdsl/tests/test_application_with_multithreading_and_pythonobjects.py b/quantdsl/tests/test_application_with_multithreading_and_pythonobjects.py index 2ee2246..f20ad0c 100644 --- a/quantdsl/tests/test_application_with_multithreading_and_pythonobjects.py +++ b/quantdsl/tests/test_application_with_multithreading_and_pythonobjects.py @@ -6,4 +6,4 @@ class TestApplicationWithPythonObjectsAndMultithreading(TestCase, ContractValuationTests): def setup_application(self): - self.app = QuantDslApplicationWithMultithreadingAndPythonObjects(num_workers=self.NUMBER_WORKERS) + self.app = QuantDslApplicationWithMultithreadingAndPythonObjects() diff --git a/quantdsl/tests/test_domain_services.py b/quantdsl/tests/test_domain_services.py index 0650837..9837f15 100644 --- a/quantdsl/tests/test_domain_services.py +++ b/quantdsl/tests/test_domain_services.py @@ -226,7 +226,8 @@ class TestSimulatedPrices(unittest.TestCase): def test_generate_simulated_prices(self, simulate_future_prices, register_simuated_price): market_calibration = Mock(spec=MarketCalibration) market_simulation = Mock(spec=MarketSimulation) - generate_simulated_prices(market_simulation=market_simulation, market_calibration=market_calibration) + prices = generate_simulated_prices(market_simulation=market_simulation, market_calibration=market_calibration) + prices = list(prices) self.assertEqual(register_simuated_price.call_count, len(simulate_future_prices.return_value)) @patch('quantdsl.domain.services.simulated_prices.get_price_process', new=lambda name: Mock( diff --git a/quantdsl/tests/test_eventsourced_repos.py b/quantdsl/tests/test_eventsourced_repos.py index 86c88f5..a9a124c 100644 --- a/quantdsl/tests/test_eventsourced_repos.py +++ b/quantdsl/tests/test_eventsourced_repos.py @@ -107,6 +107,8 @@ def test_register_simulated_price(self): assert isinstance(price, SimulatedPrice), price assert price.id simulated_price_id = make_simulated_price_id(simulation_id, '#1', price_time, price_time) + self.assertEqual(price.id, simulated_price_id) + self.app.simulated_price_repo[price.id] = price price = self.app.simulated_price_repo[simulated_price_id] assert isinstance(price, SimulatedPrice) import numpy diff --git a/quantdsl/tests/test_simulation_subscriber.py b/quantdsl/tests/test_simulation_subscriber.py index d3c6ce6..a02fef6 100644 --- a/quantdsl/tests/test_simulation_subscriber.py +++ b/quantdsl/tests/test_simulation_subscriber.py @@ -6,6 +6,7 @@ from quantdsl.domain.model.market_calibration import MarketCalibrationRepository from quantdsl.domain.model.market_simulation import MarketSimulation, MarketSimulationRepository +from quantdsl.domain.model.simulated_price import SimulatedPriceRepository from quantdsl.infrastructure.simulation_subscriber import SimulationSubscriber @@ -15,7 +16,12 @@ def setUp(self): assert_event_handlers_empty() market_simulation_repo = MagicMock(spec=MarketSimulationRepository) market_calibration_repo = MagicMock(spec=MarketCalibrationRepository) - self.simulation_subscriber = SimulationSubscriber(market_calibration_repo, market_simulation_repo) + simulated_price_repo = MagicMock(spec=SimulatedPriceRepository) + self.simulation_subscriber = SimulationSubscriber( + market_calibration_repo, + market_simulation_repo, + simulated_price_repo + ) def tearDown(self): self.simulation_subscriber.close() From f652f9d1531eb0c9176589bb284b02946c27d7e6 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 04:09:12 +0100 Subject: [PATCH 25/52] Changed queues so nodes are processed breadth first from leaves to root of the dependency graph (was more diagonal). --- quantdsl/application/with_multithreading.py | 2 +- quantdsl/domain/services/dependency_graphs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quantdsl/application/with_multithreading.py b/quantdsl/application/with_multithreading.py index b3b40bd..37ea409 100644 --- a/quantdsl/application/with_multithreading.py +++ b/quantdsl/application/with_multithreading.py @@ -7,7 +7,7 @@ class QuantDslApplicationWithMultithreading(QuantDslApplication): def __init__(self, num_threads=4, *args, **kwargs): - super(QuantDslApplicationWithMultithreading, self).__init__(call_evaluation_queue=queue.LifoQueue(), + super(QuantDslApplicationWithMultithreading, self).__init__(call_evaluation_queue=queue.Queue(), *args, **kwargs) # Start evaluation worker threads. for _ in range(num_threads): diff --git a/quantdsl/domain/services/dependency_graphs.py b/quantdsl/domain/services/dependency_graphs.py index e913a2d..26c6696 100644 --- a/quantdsl/domain/services/dependency_graphs.py +++ b/quantdsl/domain/services/dependency_graphs.py @@ -224,7 +224,7 @@ def get(self): class PythonPendingCallQueue(PendingCallQueue): def __init__(self): - self.queue = queue.LifoQueue() + self.queue = queue.Queue() def put_pending_call(self, pending_call): self.queue.put(pending_call) From 3cb079e1b2479c677e28d777ebe46bcaaf343487 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 14:25:58 +0100 Subject: [PATCH 26/52] Changed class BlackScholesPriceProcess to use a forward curve. Also changed the format of calibration params to be similar to the n-factor markovian price process code. --- README.md | 20 +++- quantdsl/priceprocess/blackscholes.py | 48 +++++--- quantdsl/tests/test_application.py | 26 ++++ quantdsl/tests/test_market_simulation.py | 19 ++- quantdsl/tests/test_price_processes.py | 146 ++++++++++++++++++----- 5 files changed, 202 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 912ff29..59d2c67 100644 --- a/README.md +++ b/README.md @@ -140,13 +140,23 @@ Calibrate from historical data. In this example, we can just register some calib price_process_name = 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess' calibration_params = { - 'GAS-LAST-PRICE': 10, - 'POWER-LAST-PRICE': 11, - 'GAS-ACTUAL-HISTORICAL-VOLATILITY': 30, - 'POWER-ACTUAL-HISTORICAL-VOLATILITY': 20, - 'GAS-POWER-CORRELATION': 0.4, + 'market': ['GAS', 'POWER'], + 'sigma': [0.3, 0.2], + 'rho': [ + [1.0, 0.4], + [0.4, 1.0], + ], + 'curve': { + 'GAS': [ + ('2011-1-1', 10), + ], + 'POWER': [ + ('2011-1-1', 11), + ], + }, } + market_calibration = app.register_market_calibration( price_process_name, calibration_params diff --git a/quantdsl/priceprocess/blackscholes.py b/quantdsl/priceprocess/blackscholes.py index a3c5bcf..f67cfc7 100644 --- a/quantdsl/priceprocess/blackscholes.py +++ b/quantdsl/priceprocess/blackscholes.py @@ -3,15 +3,38 @@ import datetime import math +import dateutil.parser import scipy import scipy.linalg +from scipy import array, sort, searchsorted from scipy.linalg import LinAlgError from quantdsl.exceptions import DslError -from quantdsl.priceprocess.base import PriceProcess +from quantdsl.priceprocess.base import PriceProcess, datetime_from_date from quantdsl.priceprocess.base import get_duration_years +class ForwardCurve(object): + def __init__(self, name, data): + self.name = name + self.data = data + self.by_date = dict( + [(datetime_from_date(dateutil.parser.parse(d)), v) for (d, v) in self.data] + ) + self.sorted = sort(array(self.by_date.keys())) + + def get_price(self, date): + try: + price = self.by_date[date] + except: + # Search for earlier date. + index = searchsorted(self.sorted, date) - 1 + if index < 0: + raise KeyError("Fixing date {} not found in '{}' forward curve.".format(date, self.name)) + price = self.by_date[self.sorted[index]] + return price + + class BlackScholesPriceProcess(PriceProcess): def simulate_future_prices(self, observation_date, requirements, path_count, calibration_params): @@ -26,15 +49,12 @@ def simulate_future_prices(self, observation_date, requirements, path_count, cal # motions, the actual historical volatility, and the last price. for commodity_name, brownian_motions in all_brownian_motions: # Get the 'last price' for this commodity. - param_name = '%s-LAST-PRICE' % commodity_name - last_price = self.get_calibration_param(param_name, calibration_params) - - # Get the 'actual historical volatility' for this commodity. - param_name = '%s-ACTUAL-HISTORICAL-VOLATILITY' % commodity_name - actual_historical_volatility = self.get_calibration_param(param_name, calibration_params) - sigma = actual_historical_volatility / 100.0 + index = calibration_params['market'].index(commodity_name) + sigma = calibration_params['sigma'][index] + curve = ForwardCurve(commodity_name, calibration_params['curve'][commodity_name]) for fixing_date, brownian_rv in brownian_motions: + last_price = curve.get_price(fixing_date) T = get_duration_years(observation_date, fixing_date) simulated_value = last_price * scipy.exp(sigma * brownian_rv - 0.5 * sigma * sigma * T) yield commodity_name, fixing_date, fixing_date, simulated_value @@ -136,15 +156,15 @@ def get_brownian_motions(self, observation_date, requirements, path_count, calib return all_brownian_motions def get_correlation_from_calibration(self, market_calibration, name_i, name_j): - market_name_pair = tuple(sorted([name_i, name_j])) - calibration_name = "%s-%s-CORRELATION" % market_name_pair + index_i = market_calibration['market'].index(name_i) + index_j = market_calibration['market'].index(name_j) try: - correlation = market_calibration[calibration_name] + correlation = market_calibration['rho'][index_i][index_j] except KeyError as e: msg = "Can't find correlation between '%s' and '%s' in market calibration params: %s: %s" % ( - market_name_pair[0], - market_name_pair[1], - market_calibration.keys(), + name_i, + name_j, + market_calibration, e ) raise DslError(msg) diff --git a/quantdsl/tests/test_application.py b/quantdsl/tests/test_application.py index 899d8b1..54aa06e 100644 --- a/quantdsl/tests/test_application.py +++ b/quantdsl/tests/test_application.py @@ -54,6 +54,32 @@ def tearDown(self): class ContractValuationTestCase(ApplicationTestCaseMixin): price_process_name = DEFAULT_PRICE_PROCESS_NAME calibration_params = { + 'market': ['#1', '#2', 'NBP', 'TTF', 'SPARKSPREAD'], + 'sigma': [0.5, 0.5, 0.5, 0.4, 0.4], + 'curve': { + '#1': [ + ('2011-1-1', 10), + ], + '#2': [ + ('2011-1-1', 10), + ], + 'NBP': [ + ('2011-1-1', 10), + ], + 'TTF': [ + ('2011-1-1', 11), + ], + 'SPARKSPREAD': [ + ('2011-1-1', 1), + ], + }, + 'rho': [ + [1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.4, 0.0], + [0.0, 0.0, 0.4, 1.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 1.0], + ], '#1-LAST-PRICE': 10, '#2-LAST-PRICE': 10, '#1-ACTUAL-HISTORICAL-VOLATILITY': 50, diff --git a/quantdsl/tests/test_market_simulation.py b/quantdsl/tests/test_market_simulation.py index af65205..84e2a93 100644 --- a/quantdsl/tests/test_market_simulation.py +++ b/quantdsl/tests/test_market_simulation.py @@ -16,11 +16,20 @@ def test_register_market_simulation(self): # Set up the market calibration. price_process_name = DEFAULT_PRICE_PROCESS_NAME calibration_params = { - '#1-LAST-PRICE': 10, - '#2-LAST-PRICE': 20, - '#1-ACTUAL-HISTORICAL-VOLATILITY': 10, - '#2-ACTUAL-HISTORICAL-VOLATILITY': 20, - '#1-#2-CORRELATION': 0.5, + 'market': ['#1', '#2'], + 'sigma': [0.1, 0.2], + 'curve': { + '#1': [ + ('2011-1-1', 10), + ], + '#2': [ + ('2011-1-1', 20), + ], + }, + 'rho': [ + [1.0, 0.5], + [0.5, 1.0], + ], } market_calibration = self.app.register_market_calibration(price_process_name, calibration_params) diff --git a/quantdsl/tests/test_price_processes.py b/quantdsl/tests/test_price_processes.py index 0cbcbca..0784338 100644 --- a/quantdsl/tests/test_price_processes.py +++ b/quantdsl/tests/test_price_processes.py @@ -16,10 +16,17 @@ def test_simulate_future_prices_no_requirements(self): observation_date=datetime.datetime(2011, 1, 1), requirements=[], path_count=2, - calibration_params={'#1-LAST-PRICE': 10, '#1-ACTUAL-HISTORICAL-VOLATILITY': 50}, + calibration_params={ + 'market': ['#1'], + 'sigma': [0.5], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ) + }, + }, )) - prices = [(p[0], p[1], p[2], p[3].all()) for p in prices] # For scipy. - self.assertEqual(prices, []) + self.assertEqual(list(prices), []) def test_simulate_future_prices_one_market_zero_volatility(self): prices = list(self.p.simulate_future_prices( @@ -30,8 +37,13 @@ def test_simulate_future_prices_one_market_zero_volatility(self): observation_date=datetime.datetime(2011, 1, 1), path_count=2, calibration_params={ - '#1-LAST-PRICE': 10, - '#1-ACTUAL-HISTORICAL-VOLATILITY': 0, + 'market': ['#1'], + 'sigma': [0.0], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ) + }, }, )) prices = [(p[0], p[1], p[2], p[3].mean()) for p in prices] # For scipy. @@ -42,15 +54,23 @@ def test_simulate_future_prices_one_market_zero_volatility(self): def test_simulate_future_prices_one_market_high_volatility(self): prices = list(self.p.simulate_future_prices( - requirements=[], + requirements=[ + ('#1', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#1', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ], observation_date=datetime.datetime(2011, 1, 1), path_count=1000, calibration_params={ - '#1-LAST-PRICE': 10, - '#1-ACTUAL-HISTORICAL-VOLATILITY': 50, + 'market': ['#1'], + 'sigma': [0.5], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ) + }, }, )) - prices = [p[2].mean() for p in prices[1:]] # For scipy. + prices = [p[3].mean() for p in prices[1:]] # For scipy. expected_prices = [10] for price, expected_price in zip(prices, expected_prices): self.assertNotEqual(price, expected_price) @@ -59,19 +79,25 @@ def test_simulate_future_prices_one_market_high_volatility(self): def test_simulate_future_prices_two_markets_zero_volatility(self): prices = list(self.p.simulate_future_prices( requirements=[ - ('#1', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1), scipy.array([ 10., 10.]).mean()), - ('#1', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2), scipy.array([ 10., 10.]).mean()), - ('#2', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1), scipy.array([ 20., 20.]).mean()), - ('#2', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2), scipy.array([ 20., 20.]).mean()), + ('#1', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#1', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ('#2', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#2', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), ], observation_date=datetime.datetime(2011, 1, 1), path_count=200000, calibration_params={ - '#1-LAST-PRICE': 10, - '#1-ACTUAL-HISTORICAL-VOLATILITY': 0, - '#2-LAST-PRICE': 20, - '#2-ACTUAL-HISTORICAL-VOLATILITY': 0, - '#1-#2-CORRELATION': 0, - }, + 'market': ['#1', '#2'], + 'sigma': [0.0, 0.0], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ), + '#2': ( + ('2011-1-1', 20), + ) + }, + 'rho': [[1, 0], [0, 1]] + } )) prices = [(p[0], p[1], p[2], p[3].mean()) for p in prices] # For scipy. self.assertEqual(prices, [ @@ -83,34 +109,88 @@ def test_simulate_future_prices_two_markets_zero_volatility(self): def test_simulate_future_prices_two_markets_high_volatility_zero_correlation(self): prices = list(self.p.simulate_future_prices( - requirements=[], + requirements=[ + ('#1', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#1', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ('#2', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#2', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ], observation_date=datetime.datetime(2011, 1, 1), path_count=1000, calibration_params={ - '#1-LAST-PRICE': 10, - '#1-ACTUAL-HISTORICAL-VOLATILITY': 50, - '#2-LAST-PRICE': 20, - '#2-ACTUAL-HISTORICAL-VOLATILITY': 50, - '#1-#2-CORRELATION': 0, + 'market': ['#1', '#2'], + 'sigma': [0.5, 0.5], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ), + '#2': ( + ('2011-1-1', 20), + ) + }, + 'rho': [[1, 0], [0, 1]] }, )) - prices = [p[2].mean() for p in prices] # For scipy. + prices = [p[3].mean() for p in prices] # For scipy. expected_prices = [10, 10, 20, 20] for price, expected_price in zip(prices, expected_prices): self.assertAlmostEqual(price, expected_price, places=0) def test_simulate_future_prices_two_markets_high_volatility_positive_correlation(self): prices = list(self.p.simulate_future_prices( - requirements=[], + requirements=[ + ('#1', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#1', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ('#2', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#2', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ], observation_date=datetime.datetime(2011, 1, 1), path_count=1000, calibration_params={ - '#1-LAST-PRICE': 10, - '#1-ACTUAL-HISTORICAL-VOLATILITY': 50, - '#2-LAST-PRICE': 20, - '#2-ACTUAL-HISTORICAL-VOLATILITY': 50, - '#1-#2-CORRELATION': 0.5, + 'market': ['#1', '#2'], + 'sigma': [0.5, 0.5], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ), + '#2': ( + ('2011-1-1', 20), + ) + }, + 'rho': [[1, 0.5], [0.5, 1]] }, )) - prices = [p[2].mean() for p in prices] # For scipy. + assert len(prices) + prices = [p[3].mean() for p in prices] # For scipy. expected_prices = [10, 10, 20, 20] for price, expected_price in zip(prices, expected_prices): self.assertAlmostEqual(price, expected_price, places=0) + + def test_simulate_future_prices_from_longer_curve(self): + prices = list(self.p.simulate_future_prices( + requirements=[ + ('#1', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#1', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ('#2', datetime.datetime(2011, 1, 1), datetime.datetime(2011, 1, 1)), + ('#2', datetime.datetime(2011, 1, 2), datetime.datetime(2011, 1, 2)), + ], + observation_date=datetime.datetime(2011, 1, 1), + path_count=1000, calibration_params={ + 'market': ['#1', '#2'], + 'sigma': [0.5, 0.5], + 'curve': { + '#1': ( + ('2011-1-1', 10), + ('2011-1-2', 15) + ), + '#2': ( + ('2011-1-1', 20), + ('2011-1-2', 25) + ) + }, + 'rho': [[1, 0.5], [0.5, 1]] + }, + )) + expected_prices = [10, 15, 20, 25] + prices = [p[3].mean() for p in prices] # For scipy. + + for price, expected_price in zip(prices, expected_prices): + self.assertAlmostEqual(price, expected_price, places=0) From ea0e531af11da84318354842f60ad58812c9284e Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 15:02:27 +0100 Subject: [PATCH 27/52] Moved class ForwardCurve. Added test for calculation of sigma. --- quantdsl/priceprocess/blackscholes.py | 55 +++++--------------------- quantdsl/priceprocess/forwardcurve.py | 25 ++++++++++++ quantdsl/tests/test_price_processes.py | 13 +++++- 3 files changed, 47 insertions(+), 46 deletions(-) create mode 100644 quantdsl/priceprocess/forwardcurve.py diff --git a/quantdsl/priceprocess/blackscholes.py b/quantdsl/priceprocess/blackscholes.py index f67cfc7..5f41e1f 100644 --- a/quantdsl/priceprocess/blackscholes.py +++ b/quantdsl/priceprocess/blackscholes.py @@ -3,36 +3,13 @@ import datetime import math -import dateutil.parser import scipy import scipy.linalg -from scipy import array, sort, searchsorted from scipy.linalg import LinAlgError from quantdsl.exceptions import DslError -from quantdsl.priceprocess.base import PriceProcess, datetime_from_date -from quantdsl.priceprocess.base import get_duration_years - - -class ForwardCurve(object): - def __init__(self, name, data): - self.name = name - self.data = data - self.by_date = dict( - [(datetime_from_date(dateutil.parser.parse(d)), v) for (d, v) in self.data] - ) - self.sorted = sort(array(self.by_date.keys())) - - def get_price(self, date): - try: - price = self.by_date[date] - except: - # Search for earlier date. - index = searchsorted(self.sorted, date) - 1 - if index < 0: - raise KeyError("Fixing date {} not found in '{}' forward curve.".format(date, self.name)) - price = self.by_date[self.sorted[index]] - return price +from quantdsl.priceprocess.base import PriceProcess, get_duration_years +from quantdsl.priceprocess.forwardcurve import ForwardCurve class BlackScholesPriceProcess(PriceProcess): @@ -59,14 +36,6 @@ def simulate_future_prices(self, observation_date, requirements, path_count, cal simulated_value = last_price * scipy.exp(sigma * brownian_rv - 0.5 * sigma * sigma * T) yield commodity_name, fixing_date, fixing_date, simulated_value - def get_calibration_param(self, param_name, calibration_params): - try: - actual_historical_volatility = calibration_params[param_name] - except KeyError: - msg = "Calibration parameter '{}' not found.".format(param_name) - raise KeyError(msg) - return actual_historical_volatility - def get_brownian_motions(self, observation_date, requirements, path_count, calibration_params): assert isinstance(observation_date, datetime.datetime), observation_date assert isinstance(requirements, list), requirements @@ -173,15 +142,11 @@ def get_correlation_from_calibration(self, market_calibration, name_i, name_j): return correlation -class BlackScholesVolatility(object): - - def calc_actual_historical_volatility(self, market, observation_date): - price_history = market.getPriceHistory(observation_date=observation_date) - prices = scipy.array([i.value for i in price_history]) - dates = [i.observation_date for i in price_history] - volatility = 100 * prices.std() / prices.mean() - duration = max(dates) - min(dates) - years = (duration.days) / 365.0 # Assumes zero seconds. - if years == 0: - raise Exception("Can't calculate volatility for price series with zero duration: %s" % (price_history)) - return float(volatility) / math.sqrt(years) +def calc_sigma(curve): + dates = scipy.array([i[0] for i in curve]) + prices = scipy.array([i[1] for i in curve]) + volatility = prices.std() / prices.mean() + duration = max(dates) - min(dates) + years = duration.total_seconds() / datetime.timedelta(365).total_seconds() + assert years > 0, "Can't calculate volatility for price series with zero days duration" + return float(volatility) / math.sqrt(years) diff --git a/quantdsl/priceprocess/forwardcurve.py b/quantdsl/priceprocess/forwardcurve.py new file mode 100644 index 0000000..05bf8b2 --- /dev/null +++ b/quantdsl/priceprocess/forwardcurve.py @@ -0,0 +1,25 @@ +import dateutil.parser +from numpy import sort, array, searchsorted + +from quantdsl.priceprocess.base import datetime_from_date + + +class ForwardCurve(object): + def __init__(self, name, data): + self.name = name + self.data = data + self.by_date = dict( + [(datetime_from_date(dateutil.parser.parse(d)), v) for (d, v) in self.data] + ) + self.sorted = sort(array(self.by_date.keys())) + + def get_price(self, date): + try: + price = self.by_date[date] + except: + # Search for earlier date. + index = searchsorted(self.sorted, date) - 1 + if index < 0: + raise KeyError("Fixing date {} not found in '{}' forward curve.".format(date, self.name)) + price = self.by_date[self.sorted[index]] + return price \ No newline at end of file diff --git a/quantdsl/tests/test_price_processes.py b/quantdsl/tests/test_price_processes.py index 0784338..68efd2f 100644 --- a/quantdsl/tests/test_price_processes.py +++ b/quantdsl/tests/test_price_processes.py @@ -3,7 +3,7 @@ import scipy -from quantdsl.priceprocess.blackscholes import BlackScholesPriceProcess +from quantdsl.priceprocess.blackscholes import BlackScholesPriceProcess, calc_sigma class TestBlackScholesPriceProcess(unittest.TestCase): @@ -194,3 +194,14 @@ def test_simulate_future_prices_from_longer_curve(self): for price, expected_price in zip(prices, expected_prices): self.assertAlmostEqual(price, expected_price, places=0) + + +class TestCalcSigma(unittest.TestCase): + def test(self): + sigma = calc_sigma([ + (datetime.datetime(2011, 1, 1), 10), + (datetime.datetime(2011, 2, 1), 11), + (datetime.datetime(2011, 3, 1), 9), + (datetime.datetime(2011, 4, 1), 10), + ]) + self.assertAlmostEqual(sigma, 0.14, places=2) From 96cfb79ef66712c8f747c5b220a9c4ff0b7a2406 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 15:18:19 +0100 Subject: [PATCH 28/52] Fixed README file. --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 59d2c67..04c4328 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Let's jump in at the deep-end with a simple model of a gas-fired power station. ```python source_code = """ -PowerStation(Date('2012-01-01'), Date('2012-01-13'), Market('GAS'), Market('POWER'), Stopped(1)) +PowerStation(Date('2012-1-1'), Date('2012-1-4'), Market('GAS'), Market('POWER'), Running()) def PowerStation(start, end, gas, power, duration_off): if (start < end): @@ -143,15 +143,21 @@ calibration_params = { 'market': ['GAS', 'POWER'], 'sigma': [0.3, 0.2], 'rho': [ - [1.0, 0.4], - [0.4, 1.0], + [1.0, 0.8], + [0.8, 1.0], ], 'curve': { 'GAS': [ - ('2011-1-1', 10), + ('2011-1-1', 12), + ('2012-1-1', 11), + ('2012-1-2', 10), + ('2012-1-3', 9), ], 'POWER': [ ('2011-1-1', 11), + ('2012-1-1', 11), + ('2012-1-2', 11), + ('2012-1-3', 11), ], }, } @@ -186,5 +192,5 @@ Inspect the estimated value. ```python estimated_value = app.get_result(valuation).result_value.mean() -assert 17 < estimated_value < 18, estimated_value +assert 3 < estimated_value < 5, estimated_value ``` From aa6f6c186560ba3a121258d30422af7049aa9f0d Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 16:10:52 +0100 Subject: [PATCH 29/52] Moved CalcAndPlot into the library. Changed readme file to use it. --- README.md | 203 ++++++------------- quantdsl/interfaces/calcandplot.py | 309 +++++++++++++++++++++++++++++ quantdsl/tests/test_readme.py | 4 +- 3 files changed, 368 insertions(+), 148 deletions(-) create mode 100644 quantdsl/interfaces/calcandplot.py diff --git a/README.md b/README.md index 04c4328..55b4c5b 100644 --- a/README.md +++ b/README.md @@ -20,38 +20,6 @@ Please register any [issues on GitHub](https://github.com/johnbywater/quantdsl/i To avoid disturbing your system's site packages, it is recommended to install into a new virtual Python environment, using [Virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/). -If you are working on a corporate LAN, with an HTTP proxy that requires authentication, then pip may fail to find -the Python Package Index. In this case you may need to download the distribution (and dependencies) by hand, and -then use the path to the downloaded files instead of the package name in the `pip` command: - -``` -pip install ~/Downloads/quantdsl-X.X.X.tar.gz -``` - -### Trouble building NumPy and SciPy? - -This package depends on [SciPy](https://www.scipy.org/), which depends on [NumPy](http://www.numpy.org/). - -Both should be automatically installed as Python dependencies, however they depend on compilers being available on -your system. - -On GNU/Linux systems you may need to install developer tools (e.g. ```build-essential``` on Debian). If -you don't want to build the binaries, you can just install NumPy and SciPy as system packages and create a Python -virtual environment that uses system site packages (`--system-site-packages`). - -Similarly, OSX users may benefit from reading [this page](https://www.scipy.org/scipylib/building/macosx.html): -install Apple’s Developer Tools, then install the Fortran compiler binary. Then you should be able -to install NumPy and SciPy using pip. - -Windows users may also not be able to install NumPy and SciPy because they do not have a -compiler installed. Obtaining one that works can be frustrating. If so, one solution would -be to install the [PythonXY](https://code.google.com/p/pythonxy/wiki/Downloads?tm=2) -distribution of Python, so that you have NumPy and SciPy - other distributions are available. -Then create a virtual environment with the `--system-site-packages` option of `virtualenv` -so that NumPy and SciPy will be available in your virtual environment. If you get bogged down, -the simpler alternative is to install *Quant DSL* directly into your PythonXY installation, -using `pip install quantdsl` or `easy_install quantdsl` if `pip` is not available. - ## Overview Quant DSL is a functional programming language for modelling derivative instruments. @@ -63,134 +31,77 @@ derivatives. The syntax of Quant DSL expressions has been [formally defined](http://www.appropriatesoftware.org/quant/docs/quant-dsl-definition-and-proof.pdf), -the semantic model is supported with mathematical proofs. The syntax is a strict subset of the Python language -syntax. This package is an implementation of the language in Python. At this time, we are not aware of any -other implementation of Quant DSL, in Python or in any other language. - - -## Usage - -To create a working program, you can copy and paste the following code snippets into a single Python file. The code -snippets in this section have been tested. Please feel free to experiment by making variations. +the semantic model is supported with mathematical proofs. -If you are using a Python virtualenv, please check that your virtualenv is activated before installing the library -and running your program. +This package is an implementation of the language in Python. -Let's jump in at the deep-end with a simple model of a gas-fired power station. - -```python -source_code = """ -PowerStation(Date('2012-1-1'), Date('2012-1-4'), Market('GAS'), Market('POWER'), Running()) - -def PowerStation(start, end, gas, power, duration_off): - if (start < end): - Wait(start, - Choice( - ProfitFromRunning(gas, power, duration_off) + PowerStation( - Tomorrow(start), end, gas, power, Running() - ), - PowerStation( - Tomorrow(start), end, gas, power, Stopped(duration_off) - ) - ) - ) - else: - return 0 - -@inline -def ProfitFromRunning(gas, power, duration_off): - if duration_off > 1: - return 0.75 * power - gas - elif duration_off == 1: - return 0.90 * power - gas - else: - return 1.00 * power - gas - -@inline -def Running(): - return 0 - -@inline -def Stopped(duration_off): - return duration_off + 1 - -@inline -def Tomorrow(today): - return today + TimeDelta('1d') -""" -``` +Function definitions are also supported, to ease construction of Quant DSL expressions. -Construct a Quant DSL application object. +The import statement is also supported to allow function definitions to be used from a library. -```python -from quantdsl.application.with_pythonobjects import QuantDslApplicationWithPythonObjects -app = QuantDslApplicationWithPythonObjects() -``` +## Usage -Compile the module into a dependency graph. +The basic steps in evaluating a model are: -```python -contract_specification = app.compile(source_code) -``` +* specification of a contract model; +* calibration of a price process; +* simulation of future prices; and +* evaluation of the contract model. -Calibrate from historical data. In this example, we can just register some calibration parameters. +A convenience function `calc_and_plot()` can be used to perform all the steps. ```python -price_process_name = 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess' - -calibration_params = { - 'market': ['GAS', 'POWER'], - 'sigma': [0.3, 0.2], - 'rho': [ - [1.0, 0.8], - [0.8, 1.0], - ], - 'curve': { - 'GAS': [ - ('2011-1-1', 12), - ('2012-1-1', 11), - ('2012-1-2', 10), - ('2012-1-3', 9), - ], - 'POWER': [ - ('2011-1-1', 11), - ('2012-1-1', 11), - ('2012-1-2', 11), - ('2012-1-3', 11), - ], - }, -} - - -market_calibration = app.register_market_calibration( - price_process_name, - calibration_params -) -``` +from quantdsl.interfaces.calcandplot import calc_and_plot -Make a simulation from the calibration. +source_code = """from quantdsl.lib.storage2 import GasStorage -```python -import datetime +GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') +""" -market_simulation = app.simulate( - contract_specification, - market_calibration, +calc_and_plot( + title="Gas Storage", + source_code=source_code, + observation_date='2011-1-1', + interest_rate=2.5, path_count=20000, - observation_date=datetime.datetime(2011, 1, 1) + perturbation_factor=0.01, + periodisation='monthly', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.5], + 'alpha': [0.1], + 'rho': [[1.0]], + 'curve': { + 'GAS': ( + ('2011-1-1', 13.5), + ('2011-2-1', 11.0), + ('2011-3-1', 10.0), + ('2011-4-1', 9.0), + ('2011-5-1', 7.5), + ('2011-6-1', 7.0), + ('2011-7-1', 6.5), + ('2011-8-1', 7.5), + ('2011-9-1', 8.5), + ('2011-10-1', 10.0), + ('2011-11-1', 11.5), + ('2011-12-1', 12.0), + ('2012-1-1', 13.5), + ('2012-2-1', 11.0), + ('2012-3-1', 10.0), + ('2012-4-1', 9.0), + ('2012-5-1', 7.5), + ('2012-6-1', 7.0), + ('2012-7-1', 6.5), + ('2012-8-1', 7.5), + ('2012-9-1', 8.5), + ('2012-10-1', 10.0), + ('2012-11-1', 11.5), + ('2012-12-1', 12.0) + ) + } + } ) -``` -Make an evaluation using the simulation. - -```python -valuation = app.evaluate(contract_specification, market_simulation) -``` - -Inspect the estimated value. - -```python -estimated_value = app.get_result(valuation).result_value.mean() -assert 3 < estimated_value < 5, estimated_value ``` diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py new file mode 100644 index 0000000..fa1b628 --- /dev/null +++ b/quantdsl/interfaces/calcandplot.py @@ -0,0 +1,309 @@ +# coding=utf-8 +from __future__ import print_function + +import collections +import datetime +import math +import sys +from threading import Event + +import dateutil.parser +import numpy +import os +from eventsourcing.domain.model.events import subscribe, unsubscribe +from matplotlib import dates as mdates, pylab as plt +from numpy import zeros + +from quantdsl.application.with_multithreading_and_python_objects import \ + QuantDslApplicationWithMultithreadingAndPythonObjects +from quantdsl.domain.model.call_result import CallResult, ResultValueComputed, make_call_result_id +from quantdsl.domain.model.contract_valuation import ContractValuation +from quantdsl.domain.model.simulated_price import make_simulated_price_id +from quantdsl.priceprocess.base import datetime_from_date + + +def calc_and_plot(*args, **kwargs): + return CalcAndPlot().run(*args, **kwargs) + + +class CalcAndPlot(object): + def __init__(self): + self.result_values_computed_count = 0 + + def run(self, title, source_code, observation_date, interest_rate, path_count, perturbation_factor, + price_process, periodisation): + app = QuantDslApplicationWithMultithreadingAndPythonObjects() + + start_compile = datetime.datetime.now() + contract_specification = app.compile(source_code) + end_compile = datetime.datetime.now() + print("Compilation in {}s".format((end_compile - start_compile).total_seconds())) + + start_calc = datetime.datetime.now() + evaluation, market_simulation = self.calc_results(app, interest_rate, observation_date, + path_count, perturbation_factor, contract_specification, + price_process['name'], price_process) + + is_evaluation_ready = Event() + + call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) + + def is_evaluation_complete(event): + return isinstance(event, CallResult.Created) and event.entity_id == call_result_id + + def on_evaluation_complete(_): + is_evaluation_ready.set() + + subscribe(is_evaluation_complete, on_evaluation_complete) + + while call_result_id not in app.call_result_repo: + if is_evaluation_ready.wait(timeout=2): + break + + unsubscribe(is_evaluation_complete, on_evaluation_complete) + + fair_value_stderr, fair_value_mean, periods = self.read_results(app, evaluation, market_simulation, + path_count) + + end_calc = datetime.datetime.now() + self.print_results(fair_value_mean, fair_value_stderr, periods) + print("") + print("Results in {}s".format((end_calc - start_calc).total_seconds())) + + if len(periods) > 1: + self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) + + def calc_results(self, app, interest_rate, observation_date, path_count, perturbation_factor, + contract_specification, price_process_name, calibration_params): + market_calibration = app.register_market_calibration( + price_process_name, + calibration_params + ) + market_simulation = app.simulate( + contract_specification, + market_calibration, + path_count=path_count, + observation_date=datetime_from_date(dateutil.parser.parse(observation_date)), + interest_rate=interest_rate, + perturbation_factor=perturbation_factor, + ) + + call_costs = app.calc_call_costs(contract_specification.id) + total_cost = sum(call_costs.values()) + + times = collections.deque() + + def is_result_value_computed(event): + return isinstance(event, ResultValueComputed) + + def count_result_values_computed(event): + times.append(datetime.datetime.now()) + if len(times) > 0.5 * total_cost: + times.popleft() + if len(times) > 1: + duration = times[-1] - times[0] + rate = len(times) / duration.total_seconds() + else: + rate = 0.001 + eta = (total_cost - self.result_values_computed_count) / rate + assert isinstance(event, ResultValueComputed) + self.result_values_computed_count += 1 + sys.stdout.write( + "\r{:.2f}% complete ({}/{}) {:.2f}/s eta {:.0f}s".format( + (100.0 * self.result_values_computed_count) / total_cost, + self.result_values_computed_count, + total_cost, + rate, + eta + ) + ) + sys.stdout.flush() + + subscribe(is_result_value_computed, count_result_values_computed) + evaluation = app.evaluate(contract_specification, market_simulation) + # unsubscribe(is_result_value_computed, count_result_values_computed) + return evaluation, market_simulation + + def read_results(self, app, evaluation, market_simulation, path_count): + assert isinstance(evaluation, ContractValuation) + + call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) + call_result = app.call_result_repo[call_result_id] + + sqrt_path_count = math.sqrt(path_count) + + fair_value = call_result.result_value + if isinstance(fair_value, (int, float, long)): + fair_value_mean = fair_value + fair_value_stderr = 0 + else: + fair_value_mean = fair_value.mean() + fair_value_stderr = fair_value.std() / sqrt_path_count + + perturbed_names = call_result.perturbed_values.keys() + perturbed_names = [i for i in perturbed_names if not i.startswith('-')] + perturbed_names = sorted(perturbed_names, key=lambda x: [int(i) for i in x.split('-')[1:]]) + + total_cash_in = zeros(path_count) + total_units = zeros(path_count) + periods = [] + for perturbed_name in perturbed_names: + + perturbed_value = call_result.perturbed_values[perturbed_name] + perturbed_value_negative = call_result.perturbed_values['-' + perturbed_name] + # Assumes format: NAME-YEAR-MONTH + perturbed_name_split = perturbed_name.split('-') + commodity_name = perturbed_name_split[0] + + if commodity_name == perturbed_name: + simulated_price_id = make_simulated_price_id(market_simulation.id, commodity_name, + market_simulation.observation_date, + market_simulation.observation_date) + + simulated_price = app.simulated_price_repo[simulated_price_id] + price_mean = simulated_price.value.mean() + price_std = simulated_price.value.std() + dy = perturbed_value - perturbed_value_negative + dx = 2 * market_simulation.perturbation_factor * simulated_price.value + contract_delta = dy / dx + hedge_units = - contract_delta + hedge_units_mean = hedge_units.mean() + hedge_units_stderr = hedge_units.std() / sqrt_path_count + cash_in = - hedge_units * simulated_price.value + cash_in_mean = cash_in.mean() + cash_in_stderr = cash_in.std() / sqrt_path_count + periods.append({ + 'commodity': perturbed_name, + 'date': None, + 'hedge_units_mean': hedge_units_mean, + 'hedge_units_stderr': hedge_units_stderr, + 'price_mean': price_mean, + 'price_std': price_std, + 'cash_in_mean': cash_in_mean, + 'cash_in_stderr': cash_in_stderr, + 'cum_cash_mean': cash_in_mean, + 'cum_cash_stderr': cash_in_stderr, + 'cum_pos_mean': hedge_units_mean, + 'cum_pos_stderr': hedge_units_stderr, + # 'total_unit_stderr': total_units_stderr, + }) + + + elif len(perturbed_name_split) > 2: + year = int(perturbed_name_split[1]) + month = int(perturbed_name_split[2]) + if len(perturbed_name_split) > 3: + day = int(perturbed_name_split[3]) + price_date = datetime.date(year, month, day) + else: + price_date = datetime.date(year, month, 1) + simulated_price_id = make_simulated_price_id(market_simulation.id, commodity_name, price_date, + price_date) + try: + simulated_price = app.simulated_price_repo[simulated_price_id] + except KeyError as e: + raise Exception("Simulated price for date {} is unavailable".format(price_date, e)) + + dy = perturbed_value - perturbed_value_negative + price = simulated_price.value + price_mean = price.mean() + dx = 2 * market_simulation.perturbation_factor * price + contract_delta = dy / dx + hedge_units = -contract_delta + cash_in = - hedge_units * price + total_units += hedge_units + total_cash_in += cash_in + hedge_units_mean = hedge_units.mean() + hedge_units_stderr = hedge_units.std() / sqrt_path_count + cash_in_stderr = cash_in.std() / sqrt_path_count + total_units_stderr = total_units.std() / sqrt_path_count + cash_in_mean = cash_in.mean() + total_units_mean = total_units.mean() + total_cash_in_mean = total_cash_in.mean() + periods.append({ + 'commodity': perturbed_name, + 'date': price_date, + 'hedge_units_mean': hedge_units_mean, + 'hedge_units_stderr': hedge_units_stderr, + 'price_mean': price_mean, + 'price_std': price.std(), # / math.sqrt(path_count), + 'cash_in_mean': cash_in_mean, + 'cash_in_stderr': cash_in_stderr, + 'cum_cash_mean': total_cash_in_mean, + 'cum_cash_stderr': total_cash_in.std() / sqrt_path_count, + 'cum_pos_mean': total_units_mean, + 'cum_pos_stderr': total_units.std() / sqrt_path_count, + 'total_unit_stderr': total_units_stderr, + }) + + return fair_value_stderr, fair_value_mean, periods + + def print_results(self, fair_value_mean, fair_value_stderr, periods): + print("") + print("") + + if periods: + for period in periods: + print(period['commodity']) + print("Price: {:.2f}".format(period['price_mean'])) + print("Hedge: {:.2f} ± {:.2f} units of {}".format(period['hedge_units_mean'], + 3 * period['hedge_units_stderr'], + period['commodity'])) + print("Cash in: {:.2f} ± {:.2f}".format(period['cash_in_mean'], 3 * period['cash_in_stderr'])) + print("Cum posn: {:.2f} ± {:.2f}".format(period['cum_pos_mean'], 3 * period['cum_pos_stderr'])) + print() + last_data = periods[-1] + print("Net cash in: {:.2f} ± {:.2f}".format(last_data['cum_cash_mean'], 3 * last_data['cum_cash_stderr'])) + print("Net position: {:.2f} ± {:.2f}".format(last_data['cum_pos_mean'], 3 * last_data['cum_pos_stderr'])) + print() + print("Fair value: {:.2f} ± {:.2f}".format(fair_value_mean, 3 * fair_value_stderr)) + + def plot_results(self, interest_rate, path_count, perturbation_factor, periods, title, periodisation): + if plt: + prices_mean = [p['price_mean'] for p in periods] + prices_std = [p['price_std'] for p in periods] + prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) + prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) + + cum_cash_mean = [p['cum_cash_mean'] for p in periods] + cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] + cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) + cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) + + cum_pos_mean = [p['cum_pos_mean'] for p in periods] + cum_pos_stderr = [p['cum_pos_stderr'] for p in periods] + cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) + cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) + + dates = [p['date'] for p in periods] + + f, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) + f.canvas.set_window_title(title) + f.suptitle('paths:{} perturbation:{} interest:{}% '.format( + path_count, perturbation_factor, interest_rate)) + if periodisation == 'monthly': + ax1.xaxis.set_major_locator(mdates.MonthLocator()) + ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + elif periodisation == 'daily': + ax1.xaxis.set_major_locator(mdates.DayLocator()) + ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + else: + raise NotImplementedError(periodisation) + + ax1.set_title('Prices') + ax1.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') + + ax2.set_title('Position') + ax2.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') + + ax3.set_title('Profit') + ax3.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') + + f.autofmt_xdate(rotation=60) + + ax1.grid() + ax2.grid() + ax3.grid() + + if not os.getenv('SUPRESS_MATPLOTLIB_PLOT_SHOW'): + plt.show() diff --git a/quantdsl/tests/test_readme.py b/quantdsl/tests/test_readme.py index adb9696..9d36794 100644 --- a/quantdsl/tests/test_readme.py +++ b/quantdsl/tests/test_readme.py @@ -45,14 +45,14 @@ def test_code_snippets_in_readme_file(self): readme_py.writelines("\n".join(lines) + '\n') # Run the code and catch errors. - p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE) + p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_MATPLOTLIB_PLOT_SHOW': 'True'}) out, err = p.communicate() out = out.decode('utf8').replace(temp_filename, readme_filename) err = err.decode('utf8').replace(temp_filename, readme_filename) exit_status = p.wait() # Check for errors running the code. - self.assertEqual(exit_status, 0, "Usage exit status {}:\n{}\n{}".format(exit_status, out, err)) + self.assertEqual(exit_status, 0, u"Usage exit status {}:\n{}\n{}".format(exit_status, out, err)) # Delete the temp file. os.unlink(temp_path) From b22e8270801f80cb73fafed963a5498f857f3a0b Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 16:22:34 +0100 Subject: [PATCH 30/52] Adjusted calc and plot, so plotting doesn't happen when unit tested. --- quantdsl/interfaces/calcandplot.py | 84 +++++++++++++++--------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index fa1b628..9ef1297 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -70,7 +70,7 @@ def on_evaluation_complete(_): print("") print("Results in {}s".format((end_calc - start_calc).total_seconds())) - if len(periods) > 1: + if plt and len(periods) > 1 and not os.getenv('SUPRESS_MATPLOTLIB_PLOT_SHOW'): self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) def calc_results(self, app, interest_rate, observation_date, path_count, perturbation_factor, @@ -259,51 +259,49 @@ def print_results(self, fair_value_mean, fair_value_stderr, periods): print("Fair value: {:.2f} ± {:.2f}".format(fair_value_mean, 3 * fair_value_stderr)) def plot_results(self, interest_rate, path_count, perturbation_factor, periods, title, periodisation): - if plt: - prices_mean = [p['price_mean'] for p in periods] - prices_std = [p['price_std'] for p in periods] - prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) - prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) - - cum_cash_mean = [p['cum_cash_mean'] for p in periods] - cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] - cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) - cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) - - cum_pos_mean = [p['cum_pos_mean'] for p in periods] - cum_pos_stderr = [p['cum_pos_stderr'] for p in periods] - cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) - cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) - - dates = [p['date'] for p in periods] - - f, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) - f.canvas.set_window_title(title) - f.suptitle('paths:{} perturbation:{} interest:{}% '.format( - path_count, perturbation_factor, interest_rate)) - if periodisation == 'monthly': - ax1.xaxis.set_major_locator(mdates.MonthLocator()) - ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) - elif periodisation == 'daily': - ax1.xaxis.set_major_locator(mdates.DayLocator()) - ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) - else: - raise NotImplementedError(periodisation) + prices_mean = [p['price_mean'] for p in periods] + prices_std = [p['price_std'] for p in periods] + prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) + prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) + + cum_cash_mean = [p['cum_cash_mean'] for p in periods] + cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] + cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) + cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) + + cum_pos_mean = [p['cum_pos_mean'] for p in periods] + cum_pos_stderr = [p['cum_pos_stderr'] for p in periods] + cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) + cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) + + dates = [p['date'] for p in periods] + + f, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) + f.canvas.set_window_title(title) + f.suptitle('paths:{} perturbation:{} interest:{}% '.format( + path_count, perturbation_factor, interest_rate)) + if periodisation == 'monthly': + ax1.xaxis.set_major_locator(mdates.MonthLocator()) + ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + elif periodisation == 'daily': + ax1.xaxis.set_major_locator(mdates.DayLocator()) + ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + else: + raise NotImplementedError(periodisation) - ax1.set_title('Prices') - ax1.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') + ax1.set_title('Prices') + ax1.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') - ax2.set_title('Position') - ax2.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') + ax2.set_title('Position') + ax2.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') - ax3.set_title('Profit') - ax3.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') + ax3.set_title('Profit') + ax3.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') - f.autofmt_xdate(rotation=60) + f.autofmt_xdate(rotation=60) - ax1.grid() - ax2.grid() - ax3.grid() + ax1.grid() + ax2.grid() + ax3.grid() - if not os.getenv('SUPRESS_MATPLOTLIB_PLOT_SHOW'): - plt.show() + plt.show() From 7302968dab6f873c62e57bf19fc2932cf88771ee Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 16:39:42 +0100 Subject: [PATCH 31/52] Added unit test for calc and plot. --- quantdsl/interfaces/calcandplot.py | 5 ++- quantdsl/tests/test_calc_and_plot.py | 58 ++++++++++++++++++++++++++++ quantdsl/tests/test_readme.py | 2 +- 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 quantdsl/tests/test_calc_and_plot.py diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index 9ef1297..b04a52f 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -31,7 +31,7 @@ def __init__(self): self.result_values_computed_count = 0 def run(self, title, source_code, observation_date, interest_rate, path_count, perturbation_factor, - price_process, periodisation): + price_process, periodisation, supress_plot=False): app = QuantDslApplicationWithMultithreadingAndPythonObjects() start_compile = datetime.datetime.now() @@ -70,7 +70,8 @@ def on_evaluation_complete(_): print("") print("Results in {}s".format((end_calc - start_calc).total_seconds())) - if plt and len(periods) > 1 and not os.getenv('SUPRESS_MATPLOTLIB_PLOT_SHOW'): + supress_plot = supress_plot or os.getenv('SUPRESS_PLOT') + if not supress_plot and plt and len(periods) > 1: self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) def calc_results(self, app, interest_rate, observation_date, path_count, perturbation_factor, diff --git a/quantdsl/tests/test_calc_and_plot.py b/quantdsl/tests/test_calc_and_plot.py new file mode 100644 index 0000000..9c9c818 --- /dev/null +++ b/quantdsl/tests/test_calc_and_plot.py @@ -0,0 +1,58 @@ +from unittest.case import TestCase + +from quantdsl.interfaces.calcandplot import calc_and_plot + + +class TestCalcAndPlot(TestCase): + def test(self): + + source_code = """from quantdsl.lib.storage2 import GasStorage + +GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') +""" + + calc_and_plot( + title="Gas Storage", + source_code=source_code, + observation_date='2011-1-1', + interest_rate=2.5, + path_count=20000, + perturbation_factor=0.01, + periodisation='monthly', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.5], + 'alpha': [0.1], + 'rho': [[1.0]], + 'curve': { + 'GAS': ( + ('2011-1-1', 13.5), + ('2011-2-1', 11.0), + ('2011-3-1', 10.0), + ('2011-4-1', 9.0), + ('2011-5-1', 7.5), + ('2011-6-1', 7.0), + ('2011-7-1', 6.5), + ('2011-8-1', 7.5), + ('2011-9-1', 8.5), + ('2011-10-1', 10.0), + ('2011-11-1', 11.5), + ('2011-12-1', 12.0), + ('2012-1-1', 13.5), + ('2012-2-1', 11.0), + ('2012-3-1', 10.0), + ('2012-4-1', 9.0), + ('2012-5-1', 7.5), + ('2012-6-1', 7.0), + ('2012-7-1', 6.5), + ('2012-8-1', 7.5), + ('2012-9-1', 8.5), + ('2012-10-1', 10.0), + ('2012-11-1', 11.5), + ('2012-12-1', 12.0) + ) + } + }, + supress_plot=True, + ) diff --git a/quantdsl/tests/test_readme.py b/quantdsl/tests/test_readme.py index 9d36794..90e70b8 100644 --- a/quantdsl/tests/test_readme.py +++ b/quantdsl/tests/test_readme.py @@ -45,7 +45,7 @@ def test_code_snippets_in_readme_file(self): readme_py.writelines("\n".join(lines) + '\n') # Run the code and catch errors. - p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_MATPLOTLIB_PLOT_SHOW': 'True'}) + p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_PLOT': 'True'}) out, err = p.communicate() out = out.decode('utf8').replace(temp_filename, readme_filename) err = err.decode('utf8').replace(temp_filename, readme_filename) From 5847c89de076ff95a509f1e93f4a0bdf4e08cede Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 16:56:50 +0100 Subject: [PATCH 32/52] Fixed unit test for calc and plot. --- quantdsl/interfaces/calcandplot.py | 126 ++++++++++++++------------- quantdsl/tests/test_calc_and_plot.py | 8 ++ 2 files changed, 72 insertions(+), 62 deletions(-) diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index b04a52f..15ab5c1 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -29,50 +29,51 @@ def calc_and_plot(*args, **kwargs): class CalcAndPlot(object): def __init__(self): self.result_values_computed_count = 0 + self.call_result_id = None + self.is_evaluation_ready = Event() + + def is_evaluation_complete(self, event): + return isinstance(event, CallResult.Created) and event.entity_id == self.call_result_id + + def on_evaluation_complete(self, _): + self.is_evaluation_ready.set() def run(self, title, source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process, periodisation, supress_plot=False): - app = QuantDslApplicationWithMultithreadingAndPythonObjects() + with QuantDslApplicationWithMultithreadingAndPythonObjects() as app: - start_compile = datetime.datetime.now() - contract_specification = app.compile(source_code) - end_compile = datetime.datetime.now() - print("Compilation in {}s".format((end_compile - start_compile).total_seconds())) + start_compile = datetime.datetime.now() + contract_specification = app.compile(source_code) + end_compile = datetime.datetime.now() + print("Compilation in {}s".format((end_compile - start_compile).total_seconds())) - start_calc = datetime.datetime.now() - evaluation, market_simulation = self.calc_results(app, interest_rate, observation_date, - path_count, perturbation_factor, contract_specification, - price_process['name'], price_process) + subscribe(self.is_evaluation_complete, self.on_evaluation_complete) + start_calc = datetime.datetime.now() - is_evaluation_ready = Event() + evaluation, market_simulation = self.calc_results(app, interest_rate, observation_date, + path_count, perturbation_factor, contract_specification, + price_process['name'], price_process) - call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) + self.call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) - def is_evaluation_complete(event): - return isinstance(event, CallResult.Created) and event.entity_id == call_result_id - def on_evaluation_complete(_): - is_evaluation_ready.set() + while self.call_result_id not in app.call_result_repo: + if self.is_evaluation_ready.wait(timeout=2): + break - subscribe(is_evaluation_complete, on_evaluation_complete) + unsubscribe(self.is_evaluation_complete, self.on_evaluation_complete) - while call_result_id not in app.call_result_repo: - if is_evaluation_ready.wait(timeout=2): - break + fair_value_stderr, fair_value_mean, periods = self.read_results(app, evaluation, market_simulation, + path_count) - unsubscribe(is_evaluation_complete, on_evaluation_complete) + end_calc = datetime.datetime.now() + self.print_results(fair_value_mean, fair_value_stderr, periods) + print("") + print("Results in {}s".format((end_calc - start_calc).total_seconds())) - fair_value_stderr, fair_value_mean, periods = self.read_results(app, evaluation, market_simulation, - path_count) - - end_calc = datetime.datetime.now() - self.print_results(fair_value_mean, fair_value_stderr, periods) - print("") - print("Results in {}s".format((end_calc - start_calc).total_seconds())) - - supress_plot = supress_plot or os.getenv('SUPRESS_PLOT') - if not supress_plot and plt and len(periods) > 1: - self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) + supress_plot = supress_plot or os.getenv('SUPRESS_PLOT') + if not supress_plot and plt and len(periods) > 1: + self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) def calc_results(self, app, interest_rate, observation_date, path_count, perturbation_factor, contract_specification, price_process_name, calibration_params): @@ -90,41 +91,42 @@ def calc_results(self, app, interest_rate, observation_date, path_count, perturb ) call_costs = app.calc_call_costs(contract_specification.id) - total_cost = sum(call_costs.values()) - - times = collections.deque() - - def is_result_value_computed(event): - return isinstance(event, ResultValueComputed) - - def count_result_values_computed(event): - times.append(datetime.datetime.now()) - if len(times) > 0.5 * total_cost: - times.popleft() - if len(times) > 1: - duration = times[-1] - times[0] - rate = len(times) / duration.total_seconds() - else: - rate = 0.001 - eta = (total_cost - self.result_values_computed_count) / rate - assert isinstance(event, ResultValueComputed) - self.result_values_computed_count += 1 - sys.stdout.write( - "\r{:.2f}% complete ({}/{}) {:.2f}/s eta {:.0f}s".format( - (100.0 * self.result_values_computed_count) / total_cost, - self.result_values_computed_count, - total_cost, - rate, - eta - ) - ) - sys.stdout.flush() + self.total_cost = sum(call_costs.values()) - subscribe(is_result_value_computed, count_result_values_computed) + self.times = collections.deque() + + subscribe(self.is_result_value_computed, self.count_result_values_computed) evaluation = app.evaluate(contract_specification, market_simulation) - # unsubscribe(is_result_value_computed, count_result_values_computed) + unsubscribe(self.is_result_value_computed, self.count_result_values_computed) return evaluation, market_simulation + def is_result_value_computed(self, event): + return isinstance(event, ResultValueComputed) + + def count_result_values_computed(self, event): + self.times.append(datetime.datetime.now()) + if len(self.times) > 0.5 * self.total_cost: + self.times.popleft() + if len(self.times) > 1: + duration = self.times[-1] - self.times[0] + rate = len(self.times) / duration.total_seconds() + else: + rate = 0.001 + eta = (self.total_cost - self.result_values_computed_count) / rate + assert isinstance(event, ResultValueComputed) + self.result_values_computed_count += 1 + sys.stdout.write( + "\r{:.2f}% complete ({}/{}) {:.2f}/s eta {:.0f}s".format( + (100.0 * self.result_values_computed_count) / self.total_cost, + self.result_values_computed_count, + self.total_cost, + rate, + eta + ) + ) + sys.stdout.flush() + + def read_results(self, app, evaluation, market_simulation, path_count): assert isinstance(evaluation, ContractValuation) diff --git a/quantdsl/tests/test_calc_and_plot.py b/quantdsl/tests/test_calc_and_plot.py index 9c9c818..5fdce7a 100644 --- a/quantdsl/tests/test_calc_and_plot.py +++ b/quantdsl/tests/test_calc_and_plot.py @@ -1,9 +1,17 @@ from unittest.case import TestCase +from eventsourcing.domain.model.events import assert_event_handlers_empty + from quantdsl.interfaces.calcandplot import calc_and_plot class TestCalcAndPlot(TestCase): + def setUp(self): + assert_event_handlers_empty() + + def tearDown(self): + assert_event_handlers_empty() + def test(self): source_code = """from quantdsl.lib.storage2 import GasStorage From a7aa7671a3bc61f566c5c4d562fe6928f1f39637 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 18:53:16 +0100 Subject: [PATCH 33/52] Fixed calc and plot. Added power station to README. --- README.md | 165 ++++++++++++++++++++++++++++- quantdsl/interfaces/calcandplot.py | 5 +- 2 files changed, 163 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 55b4c5b..75c9083 100644 --- a/README.md +++ b/README.md @@ -50,18 +50,60 @@ The basic steps in evaluating a model are: * evaluation of the contract model. A convenience function `calc_and_plot()` can be used to perform all the steps. - ```python from quantdsl.interfaces.calcandplot import calc_and_plot +``` -source_code = """from quantdsl.lib.storage2 import GasStorage +Here's an evaluation of a gas storage facility. + +```pyt hon +from quantdsl.interfaces.calcandplot import calc_and_plot -GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') -""" calc_and_plot( title="Gas Storage", - source_code=source_code, + source_code="""def GasStorage(start, end, commodity_name, quantity, target, limit, step, period): + if ((start < end) and (limit > 0)): + if quantity <= 0: + return Wait(start, Choice( + Continue(start, end, commodity_name, quantity, limit, step, period, target), + Inject(start, end, commodity_name, quantity, limit, step, period, target, 1), + )) + elif quantity >= limit: + return Wait(start, Choice( + Continue(start, end, commodity_name, quantity, limit, step, period, target), + Inject(start, end, commodity_name, quantity, limit, step, period, target, -1), + )) + else: + return Wait(start, Choice( + Continue(start, end, commodity_name, quantity, limit, step, period, target), + Inject(start, end, commodity_name, quantity, limit, step, period, target, 1), + Inject(start, end, commodity_name, quantity, limit, step, period, target, -1), + )) + else: + if target < 0 or target == quantity: + return 0 + else: + return BreachOfContract() + + +@inline +def BreachOfContract(): + -10000000000000000 + +@inline +def Continue(start, end, commodity_name, quantity, limit, step, period, target): + GasStorage(start + step, end, commodity_name, quantity, target, limit, step, period) + + +@inline +def Inject(start, end, commodity_name, quantity, limit, step, period, target, vol): + Continue(start, end, commodity_name, quantity + vol, limit, step, period, target) - \ + vol * Lift(commodity_name, period, Market(commodity_name)) + + +GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') +""", observation_date='2011-1-1', interest_rate=2.5, path_count=20000, @@ -105,3 +147,116 @@ calc_and_plot( ) ``` + +Here's an evaluation of a power station. + +```python +calc_and_plot( + title="Gas Storage", + source_code=""" +def PowerStation(start, end, gas, power, duration_off): + if (start < end): + Wait(start, + Choice( + ProfitFromRunning(gas, power, duration_off) + PowerStation( + Tomorrow(start), end, gas, power, Running() + ), + PowerStation( + Tomorrow(start), end, gas, power, Stopped(duration_off) + ) + ) + ) + else: + return 0 + +@inline +def ProfitFromRunning(gas, power, duration_off): + if duration_off > 1: + return 0.75 * power - gas + elif duration_off == 1: + return 0.90 * power - gas + else: + return 1.00 * power - gas + +@inline +def Running(): + return 0 + +@inline +def Stopped(duration_off): + return duration_off + 1 + +@inline +def Tomorrow(today): + return today + TimeDelta('1d') + +PowerStation(Date('2012-01-01'), Date('2012-01-13'), Market('GAS'), Market('POWER'), Running()) +""", + observation_date='2011-1-1', + interest_rate=2.5, + path_count=20000, + perturbation_factor=0.01, + periodisation='monthly', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS', 'POWER'], + 'sigma': [0.5, 0.3], + 'rho': [[1.0, 0.8], [0.8, 1.0]], + 'curve': { + 'GAS': [ + ('2011-1-1', 13.5), + ('2011-2-1', 11.0), + ('2011-3-1', 10.0), + ('2011-4-1', 9.0), + ('2011-5-1', 7.5), + ('2011-6-1', 7.0), + ('2011-7-1', 6.5), + ('2011-8-1', 7.5), + ('2011-9-1', 8.5), + ('2011-10-1', 10.0), + ('2011-11-1', 11.5), + ('2011-12-1', 12.0), + ('2012-1-1', 13.5), + ('2012-2-1', 11.0), + ('2012-3-1', 10.0), + ('2012-4-1', 9.0), + ('2012-5-1', 7.5), + ('2012-6-1', 7.0), + ('2012-7-1', 6.5), + ('2012-8-1', 7.5), + ('2012-9-1', 8.5), + ('2012-10-1', 10.0), + ('2012-11-1', 11.5), + ('2012-12-1', 12.0) + ], + 'POWER': [ + ('2011-1-1', 13.5), + ('2011-2-1', 11.0), + ('2011-3-1', 10.0), + ('2011-4-1', 9.0), + ('2011-5-1', 7.5), + ('2011-6-1', 7.0), + ('2011-7-1', 6.5), + ('2011-8-1', 7.5), + ('2011-9-1', 8.5), + ('2011-10-1', 10.0), + ('2011-11-1', 11.5), + ('2011-12-1', 12.0), + ('2012-1-1', 13.5), + ('2012-2-1', 11.0), + ('2012-3-1', 10.0), + ('2012-4-1', 9.0), + ('2012-5-1', 7.5), + ('2012-6-1', 7.0), + ('2012-7-1', 6.5), + ('2012-8-1', 7.5), + ('2012-9-1', 8.5), + ('2012-10-1', 10.0), + ('2012-11-1', 11.5), + ('2012-12-1', 12.0) + ] + } + } +) + +``` diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index 15ab5c1..da130c4 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -61,7 +61,6 @@ def run(self, title, source_code, observation_date, interest_rate, path_count, p if self.is_evaluation_ready.wait(timeout=2): break - unsubscribe(self.is_evaluation_complete, self.on_evaluation_complete) fair_value_stderr, fair_value_mean, periods = self.read_results(app, evaluation, market_simulation, path_count) @@ -75,6 +74,9 @@ def run(self, title, source_code, observation_date, interest_rate, path_count, p if not supress_plot and plt and len(periods) > 1: self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) + unsubscribe(self.is_result_value_computed, self.count_result_values_computed) + unsubscribe(self.is_evaluation_complete, self.on_evaluation_complete) + def calc_results(self, app, interest_rate, observation_date, path_count, perturbation_factor, contract_specification, price_process_name, calibration_params): market_calibration = app.register_market_calibration( @@ -97,7 +99,6 @@ def calc_results(self, app, interest_rate, observation_date, path_count, perturb subscribe(self.is_result_value_computed, self.count_result_values_computed) evaluation = app.evaluate(contract_specification, market_simulation) - unsubscribe(self.is_result_value_computed, self.count_result_values_computed) return evaluation, market_simulation def is_result_value_computed(self, event): From fc7fe02c1b44a4e823416235fec431cdfd649648 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 21:09:44 +0100 Subject: [PATCH 34/52] Changed calc and plot, to plot graphs for multi-markets. --- quantdsl/interfaces/calcandplot.py | 99 +++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 30 deletions(-) diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index da130c4..5dad923 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -10,9 +10,11 @@ import dateutil.parser import numpy import os + +from collections import defaultdict from eventsourcing.domain.model.events import subscribe, unsubscribe from matplotlib import dates as mdates, pylab as plt -from numpy import zeros +from numpy import zeros, cumsum from quantdsl.application.with_multithreading_and_python_objects import \ QuantDslApplicationWithMultithreadingAndPythonObjects @@ -263,49 +265,86 @@ def print_results(self, fair_value_mean, fair_value_stderr, periods): print("Fair value: {:.2f} ± {:.2f}".format(fair_value_mean, 3 * fair_value_stderr)) def plot_results(self, interest_rate, path_count, perturbation_factor, periods, title, periodisation): - prices_mean = [p['price_mean'] for p in periods] - prices_std = [p['price_std'] for p in periods] - prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) - prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) - - cum_cash_mean = [p['cum_cash_mean'] for p in periods] - cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] - cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) - cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) - - cum_pos_mean = [p['cum_pos_mean'] for p in periods] - cum_pos_stderr = [p['cum_pos_stderr'] for p in periods] - cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) - cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) - dates = [p['date'] for p in periods] + names = set([p['commodity'].split('-')[0] for p in periods]) - f, (ax1, ax2, ax3) = plt.subplots(3, sharex=True) + f, subplots = plt.subplots(1 + 2 * len(names), sharex=True) f.canvas.set_window_title(title) f.suptitle('paths:{} perturbation:{} interest:{}% '.format( path_count, perturbation_factor, interest_rate)) + if periodisation == 'monthly': - ax1.xaxis.set_major_locator(mdates.MonthLocator()) - ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + subplots[0].xaxis.set_major_locator(mdates.MonthLocator()) + subplots[0].xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) elif periodisation == 'daily': - ax1.xaxis.set_major_locator(mdates.DayLocator()) - ax1.xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + subplots[0].xaxis.set_major_locator(mdates.DayLocator()) + subplots[0].xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) else: raise NotImplementedError(periodisation) - ax1.set_title('Prices') - ax1.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') + for i, name in enumerate(names): + + _periods = [p for p in periods if p['commodity'].startswith(name)] + + dates = [p['date'] for p in _periods] + price_plot = subplots[i] + + prices_mean = [p['price_mean'] for p in _periods] + prices_std = [p['price_std'] for p in _periods] + prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) + prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) + + price_plot.set_title('Prices - {}'.format(name)) + price_plot.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') + + ymin = min(0, min(prices_minus)) - 1 + ymax = max(0, max(prices_plus)) + 1 + price_plot.set_ylim([ymin, ymax]) + + cum_pos_mean = cumsum([p['hedge_units_mean'] for p in _periods]) + cum_pos_stderr = cumsum([p['hedge_units_stderr'] for p in _periods]) + cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) + cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) - ax2.set_title('Position') - ax2.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') - ax3.set_title('Profit') - ax3.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') + pos_plot = subplots[len(names) + i] + pos_plot.set_title('Position - {}'.format(name)) + pos_plot.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') + ymin = min(0, min(cum_pos_minus)) - 1 + ymax = max(0, max(cum_pos_plus)) + 1 + pos_plot.set_ylim([ymin, ymax]) + + profit_plot = subplots[-1] + profit_plot.set_title('Profit') + + dates = [] + + cash_in_mean = defaultdict(int) + cash_in_stderr = defaultdict(int) + + for period in periods: + date = period['date'] + if date not in dates: + dates.append(date) + cash_in_mean[date] += period['cash_in_mean'] + cash_in_stderr[date] += period['cash_in_stderr'] + + + + # [cash_in[p['date']].append(p['cash_in_mean']) for p in _periods] + + cum_cash_mean = cumsum([cash_in_mean[date] for date in dates]) + cum_cash_stderr = cumsum([cash_in_stderr[date] for date in dates]) + # cum_cash_mean = [p['cum_cash_mean'] for p in periods] + # cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] + cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) + cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) + + profit_plot.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') + # profit_plot.plot(dates, cum_cash_mean, '0.25') f.autofmt_xdate(rotation=60) - ax1.grid() - ax2.grid() - ax3.grid() + [p.grid() for p in subplots] plt.show() From b125130ac190134b7963b080e13cfd997f7a441a Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 21:21:10 +0100 Subject: [PATCH 35/52] Adjusted README. --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 75c9083..a306909 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ The basic steps in evaluating a model are: * evaluation of the contract model. A convenience function `calc_and_plot()` can be used to perform all the steps. + ```python from quantdsl.interfaces.calcandplot import calc_and_plot ``` @@ -57,12 +58,11 @@ from quantdsl.interfaces.calcandplot import calc_and_plot Here's an evaluation of a gas storage facility. ```pyt hon -from quantdsl.interfaces.calcandplot import calc_and_plot - - calc_and_plot( title="Gas Storage", - source_code="""def GasStorage(start, end, commodity_name, quantity, target, limit, step, period): + + source_code=""" +def GasStorage(start, end, commodity_name, quantity, target, limit, step, period): if ((start < end) and (limit > 0)): if quantity <= 0: return Wait(start, Choice( @@ -104,11 +104,13 @@ def Inject(start, end, commodity_name, quantity, limit, step, period, target, vo GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') """, + observation_date='2011-1-1', interest_rate=2.5, path_count=20000, perturbation_factor=0.01, periodisation='monthly', + price_process={ 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', 'market': ['GAS'], @@ -153,6 +155,7 @@ Here's an evaluation of a power station. ```python calc_and_plot( title="Gas Storage", + source_code=""" def PowerStation(start, end, gas, power, duration_off): if (start < end): @@ -192,11 +195,13 @@ def Tomorrow(today): PowerStation(Date('2012-01-01'), Date('2012-01-13'), Market('GAS'), Market('POWER'), Running()) """, + observation_date='2011-1-1', interest_rate=2.5, path_count=20000, perturbation_factor=0.01, periodisation='monthly', + price_process={ 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', 'market': ['GAS', 'POWER'], From f16a0c27f3445c62863817f0ff0aaa2b457c9aef Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 21:23:47 +0100 Subject: [PATCH 36/52] Fixed plot title. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a306909..f7853bb 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Here's an evaluation of a power station. ```python calc_and_plot( - title="Gas Storage", + title="Power Station", source_code=""" def PowerStation(start, end, gas, power, duration_off): From 78b3107b2a1d8ef5f9032ae7b771405196617626 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 21:33:48 +0100 Subject: [PATCH 37/52] Adjusted README. --- README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f7853bb..7111aa7 100644 --- a/README.md +++ b/README.md @@ -40,24 +40,26 @@ Function definitions are also supported, to ease construction of Quant DSL expre The import statement is also supported to allow function definitions to be used from a library. -## Usage +## Usage Example -The basic steps in evaluating a model are: +The library provides a convenience function `calc_and_plot()` can be used to evaluate contracts. -* specification of a contract model; -* calibration of a price process; -* simulation of future prices; and -* evaluation of the contract model. - -A convenience function `calc_and_plot()` can be used to perform all the steps. +Steps for evaluating a contract include: specification of a model of a contract; calibration of a price process +for the underlying prices; simulation of future prices underlying the contract; and evaluation of the contract model +against the simulation. The library provides an application class `QuantDslApplication` which has methods that +support these steps: `compile()`, `simulate()` and `evaluate()`. The function `calc_and_plot()` uses those methods of +that application to evaluate contracts. + ```python from quantdsl.interfaces.calcandplot import calc_and_plot ``` +### Gas Storage + Here's an evaluation of a gas storage facility. -```pyt hon +```python calc_and_plot( title="Gas Storage", @@ -150,6 +152,8 @@ GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m ``` +### Power Station + Here's an evaluation of a power station. ```python From f9955ec7c91f5467f48585162b019fdf2ec370f0 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 23:23:04 +0100 Subject: [PATCH 38/52] Adjusted README. --- README.md | 111 ++------- quantdsl/interfaces/calcandplot.py | 344 +++++++++++++++------------ quantdsl/lib/powerplant2.py | 52 ++++ quantdsl/tests/test_calc_and_plot.py | 4 +- 4 files changed, 267 insertions(+), 244 deletions(-) create mode 100644 quantdsl/lib/powerplant2.py diff --git a/README.md b/README.md index 7111aa7..ed82cb1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ ## Install Use pip to install the [latest distribution](https://pypi.python.org/pypi/quantdsl) from -the Python Package Index. +the Python Package Index. To avoid disturbing your system's site packages, it is recommended to install +into a new virtual Python environment, using [Virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/). ``` pip install --upgrade pip @@ -17,8 +18,6 @@ pip install quantdsl Please register any [issues on GitHub](https://github.com/johnbywater/quantdsl/issues). -To avoid disturbing your system's site packages, it is recommended to install -into a new virtual Python environment, using [Virtualenv](http://docs.python-guide.org/en/latest/dev/virtualenvs/). ## Overview @@ -29,27 +28,32 @@ maths used in finance and trading. The elements of the language can be freely co of value. User defined functions generate extensive dependency graphs that effectively model and evaluate exotic derivatives. +## Definition and implementation + The syntax of Quant DSL expressions has been [formally defined](http://www.appropriatesoftware.org/quant/docs/quant-dsl-definition-and-proof.pdf), the semantic model is supported with mathematical proofs. -This package is an implementation of the language in Python. +This package is an implementation of the language in Python. Function definitions are also supported, to ease +construction of Quant DSL expressions. The import statement is also supported to allow function definitions to be +used from a library (see below). + +Steps for evaluating a contract include: specification of a model of a contract; calibration of a stochastic process +for the underlying prices; simulation using the price process of future prices underlying the contract; and evaluation +of the contract model against the simulation. -Function definitions are also supported, to ease construction of Quant DSL expressions. +The library provides an application class `QuantDslApplication` which +has methods that support these steps: `compile()`, `simulate()` and `evaluate()`. -The import statement is also supported to allow function definitions to be used from a library. +The library also provides a convenience function `calc()` uses those methods of that application to evaluate +contracts. -## Usage Example -The library provides a convenience function `calc_and_plot()` can be used to evaluate contracts. +## Example of usage + +The examples below use the library function `calc()` to evaluate contracts. -Steps for evaluating a contract include: specification of a model of a contract; calibration of a price process -for the underlying prices; simulation of future prices underlying the contract; and evaluation of the contract model -against the simulation. The library provides an application class `QuantDslApplication` which has methods that -support these steps: `compile()`, `simulate()` and `evaluate()`. The function `calc_and_plot()` uses those methods of -that application to evaluate contracts. - ```python from quantdsl.interfaces.calcandplot import calc_and_plot @@ -154,50 +158,15 @@ GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m ### Power Station -Here's an evaluation of a power station. +Here's an evaluation of a power station. This time, the source code imports a power station model from the library. ```python calc_and_plot( title="Power Station", - source_code=""" -def PowerStation(start, end, gas, power, duration_off): - if (start < end): - Wait(start, - Choice( - ProfitFromRunning(gas, power, duration_off) + PowerStation( - Tomorrow(start), end, gas, power, Running() - ), - PowerStation( - Tomorrow(start), end, gas, power, Stopped(duration_off) - ) - ) - ) - else: - return 0 - -@inline -def ProfitFromRunning(gas, power, duration_off): - if duration_off > 1: - return 0.75 * power - gas - elif duration_off == 1: - return 0.90 * power - gas - else: - return 1.00 * power - gas - -@inline -def Running(): - return 0 - -@inline -def Stopped(duration_off): - return duration_off + 1 - -@inline -def Tomorrow(today): - return today + TimeDelta('1d') - -PowerStation(Date('2012-01-01'), Date('2012-01-13'), Market('GAS'), Market('POWER'), Running()) + source_code="""from quantdsl.lib.powerplant2 import PowerPlant, Running + +PowerPlant(Date('2011-1-1'), Date('2011-1-6'), Running()) """, observation_date='2011-1-1', @@ -219,24 +188,6 @@ PowerStation(Date('2012-01-01'), Date('2012-01-13'), Market('GAS'), Market('POWE ('2011-4-1', 9.0), ('2011-5-1', 7.5), ('2011-6-1', 7.0), - ('2011-7-1', 6.5), - ('2011-8-1', 7.5), - ('2011-9-1', 8.5), - ('2011-10-1', 10.0), - ('2011-11-1', 11.5), - ('2011-12-1', 12.0), - ('2012-1-1', 13.5), - ('2012-2-1', 11.0), - ('2012-3-1', 10.0), - ('2012-4-1', 9.0), - ('2012-5-1', 7.5), - ('2012-6-1', 7.0), - ('2012-7-1', 6.5), - ('2012-8-1', 7.5), - ('2012-9-1', 8.5), - ('2012-10-1', 10.0), - ('2012-11-1', 11.5), - ('2012-12-1', 12.0) ], 'POWER': [ ('2011-1-1', 13.5), @@ -245,24 +196,6 @@ PowerStation(Date('2012-01-01'), Date('2012-01-13'), Market('GAS'), Market('POWE ('2011-4-1', 9.0), ('2011-5-1', 7.5), ('2011-6-1', 7.0), - ('2011-7-1', 6.5), - ('2011-8-1', 7.5), - ('2011-9-1', 8.5), - ('2011-10-1', 10.0), - ('2011-11-1', 11.5), - ('2011-12-1', 12.0), - ('2012-1-1', 13.5), - ('2012-2-1', 11.0), - ('2012-3-1', 10.0), - ('2012-4-1', 9.0), - ('2012-5-1', 7.5), - ('2012-6-1', 7.0), - ('2012-7-1', 6.5), - ('2012-8-1', 7.5), - ('2012-9-1', 8.5), - ('2012-10-1', 10.0), - ('2012-11-1', 11.5), - ('2012-12-1', 12.0) ] } } diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index 5dad923..8ab0935 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -6,6 +6,7 @@ import math import sys from threading import Event +from time import sleep import dateutil.parser import numpy @@ -24,74 +25,104 @@ from quantdsl.priceprocess.base import datetime_from_date -def calc_and_plot(*args, **kwargs): - return CalcAndPlot().run(*args, **kwargs) +def calc_and_plot(title, source_code, observation_date, periodisation, interest_rate, path_count, + perturbation_factor, price_process): + fair_value, periods = calc( + source_code=source_code, + interest_rate=interest_rate, + path_count=path_count, + observation_date=observation_date, + perturbation_factor=perturbation_factor, + price_process=price_process, + ) -class CalcAndPlot(object): - def __init__(self): + print_results(fair_value, periods, path_count) + + if periods and not os.getenv('SUPRESS_PLOT'): + plot_periods( + periods=periods, + title=title, + periodisation=periodisation, + interest_rate=interest_rate, + path_count=path_count, + perturbation_factor=perturbation_factor, + ) + + +def calc(source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process): + with Calculate(source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process) as cmd: + return cmd.run() + + +class Calculate(object): + def __init__(self, source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process): self.result_values_computed_count = 0 self.call_result_id = None - self.is_evaluation_ready = Event() + self.is_completed = Event() + subscribe(self.is_result_value_computed, self.count_result_values_computed) + subscribe(self.is_evaluation_complete, self.on_evaluation_complete) - def is_evaluation_complete(self, event): - return isinstance(event, CallResult.Created) and event.entity_id == self.call_result_id + self.source_code = source_code + self.observation_date = observation_date + self.interest_rate = interest_rate + self.path_count = path_count + self.perturbation_factor = perturbation_factor + self.price_process = price_process + self._run_once = False - def on_evaluation_complete(self, _): - self.is_evaluation_ready.set() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + unsubscribe(self.is_result_value_computed, self.count_result_values_computed) + unsubscribe(self.is_evaluation_complete, self.on_evaluation_complete) - def run(self, title, source_code, observation_date, interest_rate, path_count, perturbation_factor, - price_process, periodisation, supress_plot=False): + def run(self): + assert not self._run_once, "Already run once" + self._run_once = True with QuantDslApplicationWithMultithreadingAndPythonObjects() as app: start_compile = datetime.datetime.now() - contract_specification = app.compile(source_code) + contract_specification = app.compile(self.source_code) end_compile = datetime.datetime.now() print("Compilation in {}s".format((end_compile - start_compile).total_seconds())) - subscribe(self.is_evaluation_complete, self.on_evaluation_complete) start_calc = datetime.datetime.now() - evaluation, market_simulation = self.calc_results(app, interest_rate, observation_date, - path_count, perturbation_factor, contract_specification, - price_process['name'], price_process) + evaluation, market_simulation = self.calc_results(app, contract_specification) self.call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) while self.call_result_id not in app.call_result_repo: - if self.is_evaluation_ready.wait(timeout=2): + if self.is_completed.wait(timeout=2): break - - fair_value_stderr, fair_value_mean, periods = self.read_results(app, evaluation, market_simulation, - path_count) + fair_value, periods = self.read_results(app, evaluation, market_simulation) end_calc = datetime.datetime.now() - self.print_results(fair_value_mean, fair_value_stderr, periods) print("") print("Results in {}s".format((end_calc - start_calc).total_seconds())) - supress_plot = supress_plot or os.getenv('SUPRESS_PLOT') - if not supress_plot and plt and len(periods) > 1: - self.plot_results(interest_rate, path_count, perturbation_factor, periods, title, periodisation) - - unsubscribe(self.is_result_value_computed, self.count_result_values_computed) - unsubscribe(self.is_evaluation_complete, self.on_evaluation_complete) + return fair_value, periods - def calc_results(self, app, interest_rate, observation_date, path_count, perturbation_factor, - contract_specification, price_process_name, calibration_params): + def calc_results(self, app, contract_specification): market_calibration = app.register_market_calibration( - price_process_name, - calibration_params + self.price_process['name'], + self.price_process, ) + market_simulation = app.simulate( contract_specification, market_calibration, - path_count=path_count, - observation_date=datetime_from_date(dateutil.parser.parse(observation_date)), - interest_rate=interest_rate, - perturbation_factor=perturbation_factor, + path_count=self.path_count, + observation_date=datetime_from_date(dateutil.parser.parse(self.observation_date)), + interest_rate=self.interest_rate, + perturbation_factor=self.perturbation_factor, ) call_costs = app.calc_call_costs(contract_specification.id) @@ -99,59 +130,25 @@ def calc_results(self, app, interest_rate, observation_date, path_count, perturb self.times = collections.deque() - subscribe(self.is_result_value_computed, self.count_result_values_computed) evaluation = app.evaluate(contract_specification, market_simulation) return evaluation, market_simulation - def is_result_value_computed(self, event): - return isinstance(event, ResultValueComputed) - - def count_result_values_computed(self, event): - self.times.append(datetime.datetime.now()) - if len(self.times) > 0.5 * self.total_cost: - self.times.popleft() - if len(self.times) > 1: - duration = self.times[-1] - self.times[0] - rate = len(self.times) / duration.total_seconds() - else: - rate = 0.001 - eta = (self.total_cost - self.result_values_computed_count) / rate - assert isinstance(event, ResultValueComputed) - self.result_values_computed_count += 1 - sys.stdout.write( - "\r{:.2f}% complete ({}/{}) {:.2f}/s eta {:.0f}s".format( - (100.0 * self.result_values_computed_count) / self.total_cost, - self.result_values_computed_count, - self.total_cost, - rate, - eta - ) - ) - sys.stdout.flush() - - - def read_results(self, app, evaluation, market_simulation, path_count): + def read_results(self, app, evaluation, market_simulation): assert isinstance(evaluation, ContractValuation) call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) call_result = app.call_result_repo[call_result_id] - sqrt_path_count = math.sqrt(path_count) + sqrt_path_count = math.sqrt(self.path_count) fair_value = call_result.result_value - if isinstance(fair_value, (int, float, long)): - fair_value_mean = fair_value - fair_value_stderr = 0 - else: - fair_value_mean = fair_value.mean() - fair_value_stderr = fair_value.std() / sqrt_path_count perturbed_names = call_result.perturbed_values.keys() perturbed_names = [i for i in perturbed_names if not i.startswith('-')] perturbed_names = sorted(perturbed_names, key=lambda x: [int(i) for i in x.split('-')[1:]]) - total_cash_in = zeros(path_count) - total_units = zeros(path_count) + total_cash_in = zeros(self.path_count) + total_units = zeros(self.path_count) periods = [] for perturbed_name in perturbed_names: @@ -242,109 +239,150 @@ def read_results(self, app, evaluation, market_simulation, path_count): 'total_unit_stderr': total_units_stderr, }) - return fair_value_stderr, fair_value_mean, periods - - def print_results(self, fair_value_mean, fair_value_stderr, periods): - print("") - print("") - - if periods: - for period in periods: - print(period['commodity']) - print("Price: {:.2f}".format(period['price_mean'])) - print("Hedge: {:.2f} ± {:.2f} units of {}".format(period['hedge_units_mean'], - 3 * period['hedge_units_stderr'], - period['commodity'])) - print("Cash in: {:.2f} ± {:.2f}".format(period['cash_in_mean'], 3 * period['cash_in_stderr'])) - print("Cum posn: {:.2f} ± {:.2f}".format(period['cum_pos_mean'], 3 * period['cum_pos_stderr'])) - print() - last_data = periods[-1] - print("Net cash in: {:.2f} ± {:.2f}".format(last_data['cum_cash_mean'], 3 * last_data['cum_cash_stderr'])) - print("Net position: {:.2f} ± {:.2f}".format(last_data['cum_pos_mean'], 3 * last_data['cum_pos_stderr'])) + return fair_value, periods + + def is_result_value_computed(self, event): + return isinstance(event, ResultValueComputed) + + def count_result_values_computed(self, event): + self.times.append(datetime.datetime.now()) + if len(self.times) > 0.5 * self.total_cost: + self.times.popleft() + if len(self.times) > 1: + duration = self.times[-1] - self.times[0] + rate = len(self.times) / duration.total_seconds() + else: + rate = 0.001 + eta = (self.total_cost - self.result_values_computed_count) / rate + assert isinstance(event, ResultValueComputed) + self.result_values_computed_count += 1 + sys.stdout.write( + "\r{:.2f}% complete ({}/{}) {:.2f}/s eta {:.0f}s".format( + (100.0 * self.result_values_computed_count) / self.total_cost, + self.result_values_computed_count, + self.total_cost, + rate, + eta + ) + ) + sys.stdout.flush() + + def is_evaluation_complete(self, event): + return isinstance(event, CallResult.Created) and event.entity_id == self.call_result_id + + def on_evaluation_complete(self, _): + self.is_completed.set() + + +def print_results(fair_value, periods, path_count): + print("") + print("") + + if isinstance(fair_value, (int, float, long)): + fair_value_mean = fair_value + fair_value_stderr = 0 + else: + fair_value_mean = fair_value.mean() + fair_value_stderr = fair_value.std() / math.sqrt(path_count) + + if periods: + for period in periods: + print(period['commodity']) + print("Price: {:.2f}".format(period['price_mean'])) + print("Hedge: {:.2f} ± {:.2f} units of {}".format(period['hedge_units_mean'], + 3 * period['hedge_units_stderr'], + period['commodity'])) + print("Cash in: {:.2f} ± {:.2f}".format(period['cash_in_mean'], 3 * period['cash_in_stderr'])) + print("Cum posn: {:.2f} ± {:.2f}".format(period['cum_pos_mean'], 3 * period['cum_pos_stderr'])) print() - print("Fair value: {:.2f} ± {:.2f}".format(fair_value_mean, 3 * fair_value_stderr)) + last_data = periods[-1] + print("Net cash in: {:.2f} ± {:.2f}".format(last_data['cum_cash_mean'], 3 * last_data['cum_cash_stderr'])) + print("Net position: {:.2f} ± {:.2f}".format(last_data['cum_pos_mean'], 3 * last_data['cum_pos_stderr'])) + print() + print("Fair value: {:.2f} ± {:.2f}".format(fair_value_mean, 3 * fair_value_stderr)) - def plot_results(self, interest_rate, path_count, perturbation_factor, periods, title, periodisation): - names = set([p['commodity'].split('-')[0] for p in periods]) +def plot_periods(periods, title, periodisation, interest_rate, path_count, perturbation_factor): - f, subplots = plt.subplots(1 + 2 * len(names), sharex=True) - f.canvas.set_window_title(title) - f.suptitle('paths:{} perturbation:{} interest:{}% '.format( - path_count, perturbation_factor, interest_rate)) + names = set([p['commodity'].split('-')[0] for p in periods]) - if periodisation == 'monthly': - subplots[0].xaxis.set_major_locator(mdates.MonthLocator()) - subplots[0].xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) - elif periodisation == 'daily': - subplots[0].xaxis.set_major_locator(mdates.DayLocator()) - subplots[0].xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) - else: - raise NotImplementedError(periodisation) + f, subplots = plt.subplots(1 + 2 * len(names), sharex=True) + f.canvas.set_window_title(title) + f.suptitle('paths:{} perturbation:{} interest:{}% '.format( + path_count, perturbation_factor, interest_rate)) - for i, name in enumerate(names): + if periodisation == 'monthly': + subplots[0].xaxis.set_major_locator(mdates.MonthLocator()) + subplots[0].xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m')) + elif periodisation == 'daily': + subplots[0].xaxis.set_major_locator(mdates.DayLocator()) + subplots[0].xaxis.set_major_formatter(mdates.DateFormatter('%Y-%m-%d')) + else: + raise NotImplementedError(periodisation) - _periods = [p for p in periods if p['commodity'].startswith(name)] + for i, name in enumerate(names): - dates = [p['date'] for p in _periods] - price_plot = subplots[i] + _periods = [p for p in periods if p['commodity'].startswith(name)] - prices_mean = [p['price_mean'] for p in _periods] - prices_std = [p['price_std'] for p in _periods] - prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) - prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) + dates = [p['date'] for p in _periods] + price_plot = subplots[i] - price_plot.set_title('Prices - {}'.format(name)) - price_plot.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') + prices_mean = [p['price_mean'] for p in _periods] + prices_std = [p['price_std'] for p in _periods] + prices_plus = list(numpy.array(prices_mean) + 2 * numpy.array(prices_std)) + prices_minus = list(numpy.array(prices_mean) - 2 * numpy.array(prices_std)) - ymin = min(0, min(prices_minus)) - 1 - ymax = max(0, max(prices_plus)) + 1 - price_plot.set_ylim([ymin, ymax]) + price_plot.set_title('Prices - {}'.format(name)) + price_plot.plot(dates, prices_plus, '0.75', dates, prices_minus, '0.75', dates, prices_mean, '0.25') - cum_pos_mean = cumsum([p['hedge_units_mean'] for p in _periods]) - cum_pos_stderr = cumsum([p['hedge_units_stderr'] for p in _periods]) - cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) - cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) + ymin = min(0, min(prices_minus)) - 1 + ymax = max(0, max(prices_plus)) + 1 + price_plot.set_ylim([ymin, ymax]) + cum_pos_mean = cumsum([p['hedge_units_mean'] for p in _periods]) + cum_pos_stderr = cumsum([p['hedge_units_stderr'] for p in _periods]) + cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) + cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) - pos_plot = subplots[len(names) + i] - pos_plot.set_title('Position - {}'.format(name)) - pos_plot.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') - ymin = min(0, min(cum_pos_minus)) - 1 - ymax = max(0, max(cum_pos_plus)) + 1 - pos_plot.set_ylim([ymin, ymax]) - profit_plot = subplots[-1] - profit_plot.set_title('Profit') + pos_plot = subplots[len(names) + i] + pos_plot.set_title('Position - {}'.format(name)) + pos_plot.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') + ymin = min(0, min(cum_pos_minus)) - 1 + ymax = max(0, max(cum_pos_plus)) + 1 + pos_plot.set_ylim([ymin, ymax]) - dates = [] + profit_plot = subplots[-1] + profit_plot.set_title('Profit') - cash_in_mean = defaultdict(int) - cash_in_stderr = defaultdict(int) + dates = [] - for period in periods: - date = period['date'] - if date not in dates: - dates.append(date) - cash_in_mean[date] += period['cash_in_mean'] - cash_in_stderr[date] += period['cash_in_stderr'] + cash_in_mean = defaultdict(int) + cash_in_stderr = defaultdict(int) + + for period in periods: + date = period['date'] + if date not in dates: + dates.append(date) + cash_in_mean[date] += period['cash_in_mean'] + cash_in_stderr[date] += period['cash_in_stderr'] - # [cash_in[p['date']].append(p['cash_in_mean']) for p in _periods] + # [cash_in[p['date']].append(p['cash_in_mean']) for p in _periods] - cum_cash_mean = cumsum([cash_in_mean[date] for date in dates]) - cum_cash_stderr = cumsum([cash_in_stderr[date] for date in dates]) - # cum_cash_mean = [p['cum_cash_mean'] for p in periods] - # cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] - cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) - cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) + cum_cash_mean = cumsum([cash_in_mean[date] for date in dates]) + cum_cash_stderr = cumsum([cash_in_stderr[date] for date in dates]) + # cum_cash_mean = [p['cum_cash_mean'] for p in periods] + # cum_cash_stderr = [p['cum_cash_stderr'] for p in periods] + cum_cash_plus = list(numpy.array(cum_cash_mean) + 3 * numpy.array(cum_cash_stderr)) + cum_cash_minus = list(numpy.array(cum_cash_mean) - 3 * numpy.array(cum_cash_stderr)) - profit_plot.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') - # profit_plot.plot(dates, cum_cash_mean, '0.25') + profit_plot.plot(dates, cum_cash_plus, '0.75', dates, cum_cash_minus, '0.75', dates, cum_cash_mean, '0.25') + # profit_plot.plot(dates, cum_cash_mean, '0.25') - f.autofmt_xdate(rotation=60) + f.autofmt_xdate(rotation=60) - [p.grid() for p in subplots] + [p.grid() for p in subplots] - plt.show() + plt.show() diff --git a/quantdsl/lib/powerplant2.py b/quantdsl/lib/powerplant2.py new file mode 100644 index 0000000..b35d82b --- /dev/null +++ b/quantdsl/lib/powerplant2.py @@ -0,0 +1,52 @@ +from quantdsl.semantics import Choice, Lift, Market, TimeDelta, Wait, inline + + +def PowerPlant(start, end, duration_off): + if (start < end): + Wait(start, + Choice( + ProfitFromRunning(duration_off) + PowerPlant( + Tomorrow(start), end, Running() + ), + PowerPlant( + Tomorrow(start), end, Stopped(duration_off) + ) + ) + ) + else: + return 0 + + +@inline +def ProfitFromRunning(duration_off): + if duration_off > 1: + return 0.75 * Power() - Gas() + elif duration_off == 1: + return 0.90 * Power() - Gas() + else: + return 1.00 * Power() - Gas() + + +@inline +def Power(): + Lift('POWER', 'daily', Market('POWER')) + + +@inline +def Gas(): + Lift('GAS', 'daily', Market('GAS')) + + +@inline +def Running(): + return 0 + + +@inline +def Stopped(duration_off): + return duration_off + 1 + + +@inline +def Tomorrow(today): + return today + TimeDelta('1d') diff --git a/quantdsl/tests/test_calc_and_plot.py b/quantdsl/tests/test_calc_and_plot.py index 5fdce7a..1ea23d4 100644 --- a/quantdsl/tests/test_calc_and_plot.py +++ b/quantdsl/tests/test_calc_and_plot.py @@ -2,7 +2,7 @@ from eventsourcing.domain.model.events import assert_event_handlers_empty -from quantdsl.interfaces.calcandplot import calc_and_plot +from quantdsl.interfaces.calcandplot import calc class TestCalcAndPlot(TestCase): @@ -19,7 +19,7 @@ def test(self): GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') """ - calc_and_plot( + calc( title="Gas Storage", source_code=source_code, observation_date='2011-1-1', From 86b72d35a2b6415284b0e168703a8f947e85ba49 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Tue, 19 Sep 2017 23:27:01 +0100 Subject: [PATCH 39/52] Adjusted README. --- quantdsl/interfaces/calcandplot.py | 4 ++-- quantdsl/tests/test_calc_and_plot.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index 8ab0935..ad52ee1 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -26,7 +26,7 @@ def calc_and_plot(title, source_code, observation_date, periodisation, interest_rate, path_count, - perturbation_factor, price_process): + perturbation_factor, price_process, supress_plot=False): fair_value, periods = calc( source_code=source_code, @@ -39,7 +39,7 @@ def calc_and_plot(title, source_code, observation_date, periodisation, interest_ print_results(fair_value, periods, path_count) - if periods and not os.getenv('SUPRESS_PLOT'): + if periods and not os.getenv('SUPRESS_PLOT') and not supress_plot: plot_periods( periods=periods, title=title, diff --git a/quantdsl/tests/test_calc_and_plot.py b/quantdsl/tests/test_calc_and_plot.py index 1ea23d4..68e575f 100644 --- a/quantdsl/tests/test_calc_and_plot.py +++ b/quantdsl/tests/test_calc_and_plot.py @@ -2,7 +2,7 @@ from eventsourcing.domain.model.events import assert_event_handlers_empty -from quantdsl.interfaces.calcandplot import calc +from quantdsl.interfaces.calcandplot import calc, calc_and_plot class TestCalcAndPlot(TestCase): @@ -19,7 +19,7 @@ def test(self): GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') """ - calc( + calc_and_plot( title="Gas Storage", source_code=source_code, observation_date='2011-1-1', From 3605da271501da5b16680e5a18596d06c74b78f9 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 02:33:01 +0100 Subject: [PATCH 40/52] Added 'introduction' section to README, to explain things a bit. --- README.md | 245 ++++++++++++++++++- quantdsl/application/base.py | 5 +- quantdsl/domain/services/price_processes.py | 1 + quantdsl/domain/services/simulated_prices.py | 2 + quantdsl/interfaces/calcandplot.py | 97 +++++--- quantdsl/tests/test_application.py | 2 +- quantdsl/tests/test_calc_and_plot.py | 4 +- 7 files changed, 299 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index ed82cb1..d30f3fb 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ maths used in finance and trading. The elements of the language can be freely co of value. User defined functions generate extensive dependency graphs that effectively model and evaluate exotic derivatives. + ## Definition and implementation The syntax of Quant DSL expressions has been @@ -45,26 +46,245 @@ of the contract model against the simulation. The library provides an application class `QuantDslApplication` which has methods that support these steps: `compile()`, `simulate()` and `evaluate()`. -The library also provides a convenience function `calc()` uses those methods of that application to evaluate -contracts. +## Introduction + +Simple calculations. + +```python +from quantdsl.interfaces.calcandplot import calc + +results = calc("2 + 3 * 4 - 10 / 5") + +assert results.fair_value == 12, results.fair_value +``` + +Other binary operations. + +```python +from quantdsl.interfaces.calcandplot import calc + +results = calc("Max(9 // 2, Min(2**2, 12 % 7))") + +assert results.fair_value == 4, results.fair_value +``` + +Logical operations. + +```python +from quantdsl.interfaces.calcandplot import calc + +assert calc("1 and 2").fair_value == True +assert calc("0 and 2").fair_value == False + + +``` + +Date and time values and operations. + +```python +import datetime + +from quantdsl.interfaces.calcandplot import calc + + +results = calc("Date('2011-1-1')") +assert results.fair_value == datetime.datetime(2011, 1, 1), results.fair_value + +results = calc("Date('2011-1-10') - Date('2011-1-1')") +assert results.fair_value == datetime.timedelta(days=9), results.fair_value + +results = calc("Date('2011-1-1') + 5 * TimeDelta('1d') < Date('2011-1-10')") +assert results.fair_value == True, results.fair_value + +results = calc("Date('2011-1-1') + 10 * TimeDelta('1d') < Date('2011-1-10')") +assert results.fair_value == False, results.fair_value + +``` + +Discounting to net present value with `Settlement`. A hundred years at 2.5% gives heavy discounting from 10 to less +than 1. + +```python +results = calc( + source_code="""Settlement('2111-1-1', 10)""", + observation_date='2011-1-1', + interest_rate=2.5, +) +assert results.fair_value < 1, results.fair_value +``` -## Example of usage +If the observation date is the same as the settlement date, there is no discounting. -The examples below use the library function `calc()` to evaluate contracts. +```python +results = calc( + source_code="""Settlement('2111-1-1', 10)""", + observation_date='2111-1-1', + interest_rate=2.5, +) +assert results.fair_value == 10, results.fair_value +``` + +Underlying prices. ```python -from quantdsl.interfaces.calcandplot import calc_and_plot + +results = calc( + source_code="""Market('GAS')""", + observation_date='2011-1-1', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.0], + 'curve': { + 'GAS': [ + ('2011-1-1', 10) + ] + }, + } +) + +assert results.fair_value.mean() == 10, results.fair_value +``` + +The forward curve is used to estimate future prices, with zero-order hold from the last known value. + +```python + +results = calc( + source_code="""Fixing('2112-1-1', Market('GAS'))""", + observation_date='2011-1-1', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.0], + 'curve': { + 'GAS': [ + ('2011-1-1', 10), + ('2111-1-1', 1000) + ] + }, + }, + interest_rate=2.5, +) + +assert results.fair_value.mean() == 1000, results.fair_value.mean() +``` + +With non-zero geometric brownian motion, the future price may move. + +```python + +results = calc( + source_code="""Fixing('2112-1-1', Max(1000, Market('GAS')))""", + observation_date='2011-1-1', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.2], + 'curve': { + 'GAS': [ + ('2011-1-1', 10), + ('2111-1-1', 1000) + ] + }, + }, + interest_rate=2.5, +) + +assert results.fair_value.mean() > 1000, results.fair_value.mean() +``` + +The `Wait` element combines `Settlement` and `Fixing`. + +```python + +results = calc( + source_code="""Wait('2112-1-1', Market('GAS'))""", + observation_date='2011-1-1', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.2], + 'curve': { + 'GAS': [ + ('2011-1-1', 10), + ('2111-1-1', 1000) + ] + }, + }, + interest_rate=2.5, +) + +assert results.fair_value.mean() < 100, results.fair_value.mean() +``` + +Function definitions can be used to structure complex expressions. When evaluating an expression that involves +function calls, the call args are used to evaluating the function, which returns an expression that replaces the +function call in the expression. The expression will be evaluated with the function call arguments. + +```python +from quantdsl.interfaces.calcandplot import calc + +results = calc(source_code=""" +def Contract1(a): + a * Contract2() + 1000 * Contract3(a) + + +def Contract2(): + 25 + + +def Contract3(a): + a * 1.1 + + +Contract1(10) +""") + +assert results.fair_value == 11250, results.fair_value +``` + +Each call to a function definition becomes a node on a dependecy graph. Each call is internally +memoised, so it is only called once with the same argument values, and the result of such a call is reused. + +The function body can be an if-else block, so that the expression returned depends upon the function call argument +values. + +```python +from quantdsl.interfaces.calcandplot import calc + +results = calc(source_code=""" +def Fib(n): + if n > 1: + Fib(n-1) + Fib(n-2) + else: + n + +Fib(60) +""") + +assert results.fair_value == 1548008755920, results.fair_value +``` + +## Examples of usage + +The examples below use the library function `calc_print_plot()` to evaluate contracts. + +```python +from quantdsl.interfaces.calcandplot import calc_print_plot ``` ### Gas Storage Here's an evaluation of a gas storage facility. +This example uses a forward curve that reflects seasonal variations across the term of the contract. + ```python -calc_and_plot( +results = calc_print_plot( title="Gas Storage", source_code=""" @@ -93,10 +313,6 @@ def GasStorage(start, end, commodity_name, quantity, target, limit, step, period return BreachOfContract() -@inline -def BreachOfContract(): - -10000000000000000 - @inline def Continue(start, end, commodity_name, quantity, limit, step, period, target): GasStorage(start + step, end, commodity_name, quantity, target, limit, step, period) @@ -108,6 +324,11 @@ def Inject(start, end, commodity_name, quantity, limit, step, period, target, vo vol * Lift(commodity_name, period, Market(commodity_name)) +@inline +def BreachOfContract(): + -10000000000000000 + + GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') """, @@ -160,8 +381,10 @@ GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m Here's an evaluation of a power station. This time, the source code imports a power station model from the library. +This example uses a market model with two correlated markets. + ```python -calc_and_plot( +results = calc_print_plot( title="Power Station", source_code="""from quantdsl.lib.powerplant2 import PowerPlant, Running diff --git a/quantdsl/application/base.py b/quantdsl/application/base.py index 5620d1f..58cb658 100644 --- a/quantdsl/application/base.py +++ b/quantdsl/application/base.py @@ -152,7 +152,6 @@ def identify_simulation_requirements(self, contract_specification, observation_d requirements) def start_contract_valuation(self, contract_specification_id, market_simulation_id): - assert isinstance(contract_specification_id, six.string_types), contract_specification_id return start_contract_valuation(contract_specification_id, market_simulation_id) def loop_on_evaluation_queue(self): @@ -201,8 +200,8 @@ def simulate(self, contract_specification, market_calibration, observation_date, ) return market_simulation - def evaluate(self, contract_specification, market_simulation): - return self.start_contract_valuation(contract_specification.id, market_simulation.id) + def evaluate(self, contract_specification_id, market_simulation_id): + return self.start_contract_valuation(contract_specification_id, market_simulation_id) def get_result(self, contract_valuation): call_result_id = make_call_result_id(contract_valuation.id, contract_valuation.contract_specification_id) diff --git a/quantdsl/domain/services/price_processes.py b/quantdsl/domain/services/price_processes.py index f7a7cb4..a24b887 100644 --- a/quantdsl/domain/services/price_processes.py +++ b/quantdsl/domain/services/price_processes.py @@ -4,6 +4,7 @@ def get_price_process(price_process_name): # Load the price process object. + assert price_process_name, "Price process name is required" price_process_module_name, price_process_class_name = price_process_name.rsplit('.', 1) try: price_process_module = __import__(price_process_module_name, '', '', '*') diff --git a/quantdsl/domain/services/simulated_prices.py b/quantdsl/domain/services/simulated_prices.py index 149fac6..02d6e7c 100644 --- a/quantdsl/domain/services/simulated_prices.py +++ b/quantdsl/domain/services/simulated_prices.py @@ -20,6 +20,8 @@ def generate_simulated_prices(market_simulation, market_calibration): def simulate_future_prices(market_simulation, market_calibration): assert isinstance(market_simulation, MarketSimulation), market_simulation + if not market_simulation.requirements: + return [] assert isinstance(market_calibration, MarketCalibration), market_calibration price_process = get_price_process(market_calibration.price_process_name) assert isinstance(price_process, PriceProcess), price_process diff --git a/quantdsl/interfaces/calcandplot.py b/quantdsl/interfaces/calcandplot.py index ad52ee1..ba9b43c 100644 --- a/quantdsl/interfaces/calcandplot.py +++ b/quantdsl/interfaces/calcandplot.py @@ -4,18 +4,16 @@ import collections import datetime import math +import os import sys +from collections import defaultdict from threading import Event -from time import sleep import dateutil.parser import numpy -import os - -from collections import defaultdict from eventsourcing.domain.model.events import subscribe, unsubscribe from matplotlib import dates as mdates, pylab as plt -from numpy import zeros, cumsum +from numpy import cumsum, zeros from quantdsl.application.with_multithreading_and_python_objects import \ QuantDslApplicationWithMultithreadingAndPythonObjects @@ -25,33 +23,47 @@ from quantdsl.priceprocess.base import datetime_from_date -def calc_and_plot(title, source_code, observation_date, periodisation, interest_rate, path_count, - perturbation_factor, price_process, supress_plot=False): +class Results(object): + def __init__(self, fair_value, periods): + self.fair_value = fair_value + self.periods = periods - fair_value, periods = calc( - source_code=source_code, - interest_rate=interest_rate, - path_count=path_count, - observation_date=observation_date, - perturbation_factor=perturbation_factor, - price_process=price_process, - ) - print_results(fair_value, periods, path_count) +def calc_print_plot(title, source_code, observation_date, periodisation, interest_rate, path_count, + perturbation_factor, price_process, supress_plot=False): - if periods and not os.getenv('SUPRESS_PLOT') and not supress_plot: + results = calc_print(source_code, observation_date, interest_rate, path_count, perturbation_factor, + price_process) + + if results.periods and not supress_plot and not os.getenv('SUPRESS_PLOT'): plot_periods( - periods=periods, + periods=results.periods, title=title, periodisation=periodisation, interest_rate=interest_rate, path_count=path_count, perturbation_factor=perturbation_factor, ) + return results -def calc(source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process): - with Calculate(source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process) as cmd: +def calc_print(source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process): + results = calc( + source_code=source_code, + interest_rate=interest_rate, + path_count=path_count, + observation_date=observation_date, + perturbation_factor=perturbation_factor, + price_process=price_process, + ) + print_results(results, path_count) + return results + + +def calc(source_code, observation_date=None, interest_rate=0, path_count=20000, perturbation_factor=0.01, + price_process=None): + with Calculate(source_code, observation_date, interest_rate, path_count, perturbation_factor, price_process) as \ + cmd: return cmd.run() @@ -97,30 +109,39 @@ def run(self): self.call_result_id = make_call_result_id(evaluation.id, evaluation.contract_specification_id) - while self.call_result_id not in app.call_result_repo: if self.is_completed.wait(timeout=2): break - fair_value, periods = self.read_results(app, evaluation, market_simulation) + results = self.read_results(app, evaluation, market_simulation) end_calc = datetime.datetime.now() print("") print("Results in {}s".format((end_calc - start_calc).total_seconds())) - return fair_value, periods + return results def calc_results(self, app, contract_specification): + if self.price_process is not None: + price_process_name = self.price_process['name'] + calibration_params = {k: v for k, v in self.price_process.items() if k != 'name'} + else: + price_process_name = 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess' + calibration_params = {} market_calibration = app.register_market_calibration( - self.price_process['name'], - self.price_process, + price_process_name=price_process_name, + calibration_params=calibration_params ) + if self.observation_date is not None: + observation_date = datetime_from_date(dateutil.parser.parse(self.observation_date)) + else: + observation_date = None market_simulation = app.simulate( contract_specification, market_calibration, path_count=self.path_count, - observation_date=datetime_from_date(dateutil.parser.parse(self.observation_date)), + observation_date=observation_date, interest_rate=self.interest_rate, perturbation_factor=self.perturbation_factor, ) @@ -130,7 +151,7 @@ def calc_results(self, app, contract_specification): self.times = collections.deque() - evaluation = app.evaluate(contract_specification, market_simulation) + evaluation = app.evaluate(contract_specification.id, market_simulation.id) return evaluation, market_simulation def read_results(self, app, evaluation, market_simulation): @@ -239,7 +260,7 @@ def read_results(self, app, evaluation, market_simulation): 'total_unit_stderr': total_units_stderr, }) - return fair_value, periods + return Results(fair_value, periods) def is_result_value_computed(self, event): return isinstance(event, ResultValueComputed) @@ -274,19 +295,19 @@ def on_evaluation_complete(self, _): self.is_completed.set() -def print_results(fair_value, periods, path_count): +def print_results(results, path_count): print("") print("") - if isinstance(fair_value, (int, float, long)): - fair_value_mean = fair_value + if isinstance(results.fair_value, (int, float, long)): + fair_value_mean = results.fair_value fair_value_stderr = 0 else: - fair_value_mean = fair_value.mean() - fair_value_stderr = fair_value.std() / math.sqrt(path_count) + fair_value_mean = results.fair_value.mean() + fair_value_stderr = results.fair_value.std() / math.sqrt(path_count) - if periods: - for period in periods: + if results.periods: + for period in results.periods: print(period['commodity']) print("Price: {:.2f}".format(period['price_mean'])) print("Hedge: {:.2f} ± {:.2f} units of {}".format(period['hedge_units_mean'], @@ -295,7 +316,7 @@ def print_results(fair_value, periods, path_count): print("Cash in: {:.2f} ± {:.2f}".format(period['cash_in_mean'], 3 * period['cash_in_stderr'])) print("Cum posn: {:.2f} ± {:.2f}".format(period['cum_pos_mean'], 3 * period['cum_pos_stderr'])) print() - last_data = periods[-1] + last_data = results.periods[-1] print("Net cash in: {:.2f} ± {:.2f}".format(last_data['cum_cash_mean'], 3 * last_data['cum_cash_stderr'])) print("Net position: {:.2f} ± {:.2f}".format(last_data['cum_pos_mean'], 3 * last_data['cum_pos_stderr'])) print() @@ -303,7 +324,6 @@ def print_results(fair_value, periods, path_count): def plot_periods(periods, title, periodisation, interest_rate, path_count, perturbation_factor): - names = set([p['commodity'].split('-')[0] for p in periods]) f, subplots = plt.subplots(1 + 2 * len(names), sharex=True) @@ -344,7 +364,6 @@ def plot_periods(periods, title, periodisation, interest_rate, path_count, pertu cum_pos_plus = list(numpy.array(cum_pos_mean) + 3 * numpy.array(cum_pos_stderr)) cum_pos_minus = list(numpy.array(cum_pos_mean) - 3 * numpy.array(cum_pos_stderr)) - pos_plot = subplots[len(names) + i] pos_plot.set_title('Position - {}'.format(name)) pos_plot.plot(dates, cum_pos_plus, '0.75', dates, cum_pos_minus, '0.75', dates, cum_pos_mean, '0.25') @@ -367,8 +386,6 @@ def plot_periods(periods, title, periodisation, interest_rate, path_count, pertu cash_in_mean[date] += period['cash_in_mean'] cash_in_stderr[date] += period['cash_in_stderr'] - - # [cash_in[p['date']].append(p['cash_in_mean']) for p in _periods] cum_cash_mean = cumsum([cash_in_mean[date] for date in dates]) diff --git a/quantdsl/tests/test_application.py b/quantdsl/tests/test_application.py index 54aa06e..2051bb9 100644 --- a/quantdsl/tests/test_application.py +++ b/quantdsl/tests/test_application.py @@ -130,7 +130,7 @@ def assert_contract_value(self, specification, expected_value, expected_deltas=N # call_result_listener = None # Start the contract valuation. - contract_valuation = self.app.evaluate(contract_specification, market_simulation) + contract_valuation = self.app.evaluate(contract_specification.id, market_simulation.id) assert isinstance(contract_valuation, ContractValuation) # # Get the call result. diff --git a/quantdsl/tests/test_calc_and_plot.py b/quantdsl/tests/test_calc_and_plot.py index 68e575f..cbb8daf 100644 --- a/quantdsl/tests/test_calc_and_plot.py +++ b/quantdsl/tests/test_calc_and_plot.py @@ -2,7 +2,7 @@ from eventsourcing.domain.model.events import assert_event_handlers_empty -from quantdsl.interfaces.calcandplot import calc, calc_and_plot +from quantdsl.interfaces.calcandplot import calc, calc_print_plot class TestCalcAndPlot(TestCase): @@ -19,7 +19,7 @@ def test(self): GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') """ - calc_and_plot( + calc_print_plot( title="Gas Storage", source_code=source_code, observation_date='2011-1-1', From a80d7aac4d7a126549d507df94fc3ed6ddd30220 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 02:36:59 +0100 Subject: [PATCH 41/52] Removed excessive import statements. --- README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/README.md b/README.md index d30f3fb..2fe124e 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,6 @@ assert results.fair_value == 12, results.fair_value Other binary operations. ```python -from quantdsl.interfaces.calcandplot import calc - results = calc("Max(9 // 2, Min(2**2, 12 % 7))") assert results.fair_value == 4, results.fair_value @@ -72,8 +70,6 @@ assert results.fair_value == 4, results.fair_value Logical operations. ```python -from quantdsl.interfaces.calcandplot import calc - assert calc("1 and 2").fair_value == True assert calc("0 and 2").fair_value == False @@ -85,8 +81,6 @@ Date and time values and operations. ```python import datetime -from quantdsl.interfaces.calcandplot import calc - results = calc("Date('2011-1-1')") assert results.fair_value == datetime.datetime(2011, 1, 1), results.fair_value @@ -226,8 +220,6 @@ function calls, the call args are used to evaluating the function, which returns function call in the expression. The expression will be evaluated with the function call arguments. ```python -from quantdsl.interfaces.calcandplot import calc - results = calc(source_code=""" def Contract1(a): a * Contract2() + 1000 * Contract3(a) @@ -254,8 +246,6 @@ The function body can be an if-else block, so that the expression returned depen values. ```python -from quantdsl.interfaces.calcandplot import calc - results = calc(source_code=""" def Fib(n): if n > 1: From ba7fccbcdcf8728ac9bc871de48a982d5813b6dd Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 02:43:54 +0100 Subject: [PATCH 42/52] Adjusted README. --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2fe124e..c17c76d 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ results = calc( assert results.fair_value < 1, results.fair_value ``` -If the observation date is the same as the settlement date, there is no discounting. +If the effective present time is the same as the settlement date, there is no discounting. ```python results = calc( @@ -117,7 +117,13 @@ results = calc( observation_date='2111-1-1', interest_rate=2.5, ) +assert results.fair_value == 10, results.fair_value +results = calc( + source_code="""Fixing('2111-1-1', Settlement('2111-1-1', 10))""", + observation_date='2011-1-1', + interest_rate=2.5, +) assert results.fair_value == 10, results.fair_value ``` @@ -212,6 +218,25 @@ results = calc( interest_rate=2.5, ) +assert results.fair_value.mean() < 100, results.fair_value.mean() + +results = calc( + source_code="""Settlement('2112-1-1', Fixing('2112-1-1', Market('GAS')))""", + observation_date='2011-1-1', + price_process={ + 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', + 'market': ['GAS'], + 'sigma': [0.2], + 'curve': { + 'GAS': [ + ('2011-1-1', 10), + ('2111-1-1', 1000) + ] + }, + }, + interest_rate=2.5, +) + assert results.fair_value.mean() < 100, results.fair_value.mean() ``` From 636e12772b4a8350e894ddb996fe55ba3974ebea Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 03:13:38 +0100 Subject: [PATCH 43/52] Adjusted README. --- README.md | 94 +++++++++++++++++++++++-------------------------------- 1 file changed, 40 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index c17c76d..8653600 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,6 @@ Logical operations. ```python assert calc("1 and 2").fair_value == True assert calc("0 and 2").fair_value == False - - ``` Date and time values and operations. @@ -93,41 +91,10 @@ assert results.fair_value == True, results.fair_value results = calc("Date('2011-1-1') + 10 * TimeDelta('1d') < Date('2011-1-10')") assert results.fair_value == False, results.fair_value - -``` - -Discounting to net present value with `Settlement`. A hundred years at 2.5% gives heavy discounting from 10 to less -than 1. - -```python -results = calc( - source_code="""Settlement('2111-1-1', 10)""", - observation_date='2011-1-1', - interest_rate=2.5, -) - -assert results.fair_value < 1, results.fair_value -``` - -If the effective present time is the same as the settlement date, there is no discounting. - -```python -results = calc( - source_code="""Settlement('2111-1-1', 10)""", - observation_date='2111-1-1', - interest_rate=2.5, -) -assert results.fair_value == 10, results.fair_value - -results = calc( - source_code="""Fixing('2111-1-1', Settlement('2111-1-1', 10))""", - observation_date='2011-1-1', - interest_rate=2.5, -) -assert results.fair_value == 10, results.fair_value ``` -Underlying prices. +Underlying prices are expressed with the `Market` element. A price process is used to simulate future +prices. ```python @@ -149,6 +116,9 @@ results = calc( assert results.fair_value.mean() == 10, results.fair_value ``` +The `Fixing` element is used to condition the effective present time of included expressions. In the example below, +the expression evaluates to the 'GAS' market price on '2112-1-1'. + The forward curve is used to estimate future prices, with zero-order hold from the last known value. ```python @@ -173,7 +143,7 @@ results = calc( assert results.fair_value.mean() == 1000, results.fair_value.mean() ``` -With non-zero geometric brownian motion, the future price may move. +With geometric brownian motion, there may be future price movements. ```python @@ -197,31 +167,47 @@ results = calc( assert results.fair_value.mean() > 1000, results.fair_value.mean() ``` -The `Wait` element combines `Settlement` and `Fixing`. +Discounting to net present value with `Settlement`. A hundred years at 2.5% gives heavy discounting from 10 to less +than 1. ```python +results = calc( + source_code="""Settlement('2111-1-1', 10)""", + observation_date='2011-1-1', + interest_rate=2.5, +) + +assert results.fair_value < 1, results.fair_value +``` +If the effective present time is the same as the settlement date, there is no discounting. + +```python results = calc( - source_code="""Wait('2112-1-1', Market('GAS'))""", + source_code="""Settlement('2111-1-1', 10)""", + observation_date='2111-1-1', + interest_rate=2.5, +) +assert results.fair_value == 10, results.fair_value + +results = calc( + source_code="""Fixing('2111-1-1', Settlement('2111-1-1', 10))""", observation_date='2011-1-1', - price_process={ - 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', - 'market': ['GAS'], - 'sigma': [0.2], - 'curve': { - 'GAS': [ - ('2011-1-1', 10), - ('2111-1-1', 1000) - ] - }, - }, interest_rate=2.5, ) +assert results.fair_value == 10, results.fair_value +``` -assert results.fair_value.mean() < 100, results.fair_value.mean() + +The `Wait` element combines `Settlement` and `Fixing`, so that a single date value is used both to condition the +effective present time of the included expression, and also the value of that expression is discounted to the +effective present time of the including expression. + +```python results = calc( - source_code="""Settlement('2112-1-1', Fixing('2112-1-1', Market('GAS')))""", + # source_code="""Settlement('2112-1-1', Fixing('2112-1-1', Market('GAS')))""", + source_code="""Wait('2112-1-1', Market('GAS'))""", observation_date='2011-1-1', price_process={ 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', @@ -264,12 +250,12 @@ Contract1(10) assert results.fair_value == 11250, results.fair_value ``` -Each call to a function definition becomes a node on a dependecy graph. Each call is internally -memoised, so it is only called once with the same argument values, and the result of such a call is reused. - The function body can be an if-else block, so that the expression returned depends upon the function call argument values. +Each call to a (non-inlined) function definition becomes a node on a dependency graph. Each call is internally +memoised, so it is only called once with the same argument values, and the result of such a call is reused. + ```python results = calc(source_code=""" def Fib(n): From 37efde62815a12a91fc26e2df76d984ef2d6d1f9 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:02:22 +0100 Subject: [PATCH 44/52] Adjusted README. --- README.md | 135 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 76 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 8653600..faa928e 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,54 @@ results = calc("Date('2011-1-1') + 10 * TimeDelta('1d') < Date('2011-1-10')") assert results.fair_value == False, results.fair_value ``` +### Function definitions + +Function definitions can be used to structure complex expressions. When evaluating an expression that involves +function calls, the call args are used to evaluating the function, which returns an expression that replaces the +function call in the expression. The expression will be evaluated with the function call arguments. + +```python +results = calc(source_code=""" +def Contract1(a): + a * Contract2() + 1000 * Contract3(a) + + +def Contract2(): + 25 + + +def Contract3(a): + a * 1.1 + + +Contract1(10) +""") + +assert results.fair_value == 11250, results.fair_value +``` + +The function body can be an if-else block, so that the expression returned depends upon the function call argument +values. + +Each call to a (non-inlined) function definition becomes a node on a dependency graph. Each call is internally +memoised, so it is only called once with the same argument values, and the result of such a call is reused. + +```python +results = calc(source_code=""" +def Fib(n): + if n > 1: + Fib(n-1) + Fib(n-2) + else: + n + +Fib(60) +""") + +assert results.fair_value == 1548008755920, results.fair_value +``` + +### Market + Underlying prices are expressed with the `Market` element. A price process is used to simulate future prices. @@ -116,6 +164,8 @@ results = calc( assert results.fair_value.mean() == 10, results.fair_value ``` +### Fixing + The `Fixing` element is used to condition the effective present time of included expressions. In the example below, the expression evaluates to the 'GAS' market price on '2112-1-1'. @@ -167,6 +217,8 @@ results = calc( assert results.fair_value.mean() > 1000, results.fair_value.mean() ``` +### Settlement + Discounting to net present value with `Settlement`. A hundred years at 2.5% gives heavy discounting from 10 to less than 1. @@ -198,6 +250,7 @@ results = calc( assert results.fair_value == 10, results.fair_value ``` +### Wait The `Wait` element combines `Settlement` and `Fixing`, so that a single date value is used both to condition the effective present time of the included expression, and also the value of that expression is discounted to the @@ -226,53 +279,15 @@ results = calc( assert results.fair_value.mean() < 100, results.fair_value.mean() ``` -Function definitions can be used to structure complex expressions. When evaluating an expression that involves -function calls, the call args are used to evaluating the function, which returns an expression that replaces the -function call in the expression. The expression will be evaluated with the function call arguments. - -```python -results = calc(source_code=""" -def Contract1(a): - a * Contract2() + 1000 * Contract3(a) - - -def Contract2(): - 25 - - -def Contract3(a): - a * 1.1 - - -Contract1(10) -""") - -assert results.fair_value == 11250, results.fair_value -``` - -The function body can be an if-else block, so that the expression returned depends upon the function call argument -values. - -Each call to a (non-inlined) function definition becomes a node on a dependency graph. Each call is internally -memoised, so it is only called once with the same argument values, and the result of such a call is reused. - -```python -results = calc(source_code=""" -def Fib(n): - if n > 1: - Fib(n-1) + Fib(n-2) - else: - n - -Fib(60) -""") - -assert results.fair_value == 1548008755920, results.fair_value -``` ## Examples of usage -The examples below use the library function `calc_print_plot()` to evaluate contracts. +The examples below use the library function `calc_print_plot()` to evaluate contracts. If you run these examples, +the deltas for each market in each period are calculated, and risk neutral hedge positions are printed for each market + in each period, along with the overall fair value. A plot is displayed showing underlying prices, the cumulative hedge positions, and the cummulate cash position from the hedge positions. + The plot shows the statistical distribution of the simulated prices, and the statistical error of the hedge + positions and the cash flow. Comparing the resulting net cash position with the fair value gives an indication of + how well the deltas are performing. ```python from quantdsl.interfaces.calcandplot import calc_print_plot @@ -330,7 +345,7 @@ def BreachOfContract(): -10000000000000000 -GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') +GasStorage(Date('2011-6-1'), Date('2011-12-1'), 'GAS', 0, 0, 50000, TimeDelta('1m'), 'monthly') """, observation_date='2011-1-1', @@ -376,6 +391,7 @@ GasStorage(Date('2011-6-1'), Date('2011-9-1'), 'GAS', 0, 0, 50000, TimeDelta('1m } ) +assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ``` ### Power Station @@ -390,14 +406,14 @@ results = calc_print_plot( source_code="""from quantdsl.lib.powerplant2 import PowerPlant, Running -PowerPlant(Date('2011-1-1'), Date('2011-1-6'), Running()) +PowerPlant(Date('2012-1-1'), Date('2012-1-6'), Running()) """, observation_date='2011-1-1', interest_rate=2.5, path_count=20000, perturbation_factor=0.01, - periodisation='monthly', + periodisation='daily', price_process={ 'name': 'quantdsl.priceprocess.blackscholes.BlackScholesPriceProcess', @@ -406,23 +422,24 @@ PowerPlant(Date('2011-1-1'), Date('2011-1-6'), Running()) 'rho': [[1.0, 0.8], [0.8, 1.0]], 'curve': { 'GAS': [ - ('2011-1-1', 13.5), - ('2011-2-1', 11.0), - ('2011-3-1', 10.0), - ('2011-4-1', 9.0), - ('2011-5-1', 7.5), - ('2011-6-1', 7.0), + ('2011-1-1', 13.0), + ('2012-1-1', 13.0), + ('2012-1-2', 13.1), + ('2012-1-3', 12.8), + ('2012-1-4', 15.9), + ('2012-1-5', 13.1), ], 'POWER': [ - ('2011-1-1', 13.5), - ('2011-2-1', 11.0), - ('2011-3-1', 10.0), - ('2011-4-1', 9.0), - ('2011-5-1', 7.5), - ('2011-6-1', 7.0), + ('2011-1-1', 2.5), + ('2012-1-1', 5.6), + ('2012-1-2', 5.6), + ('2012-1-3', 12.9), + ('2012-1-4', 26.9), + ('2012-1-5', 1.8), ] } } ) +assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ``` From 478d5717c733d1eac167f88ae92a8bc809b4a114 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:12:33 +0100 Subject: [PATCH 45/52] Adjusted README. --- README.md | 9 ++++++--- quantdsl/tests/test_readme.py | 7 +++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index faa928e..93c6bd6 100644 --- a/README.md +++ b/README.md @@ -95,9 +95,12 @@ assert results.fair_value == False, results.fair_value ### Function definitions -Function definitions can be used to structure complex expressions. When evaluating an expression that involves -function calls, the call args are used to evaluating the function, which returns an expression that replaces the -function call in the expression. The expression will be evaluated with the function call arguments. +Function definitions can be used to structure complex expressions. The call args of the function definition +can be used as names in the function definition's expressions. + +When evaluating an expression that involves calls to function definitions, the call to the function definition is +firstly replaced with the expression returned by the function definition, so that a larger expression is formed. + ```python results = calc(source_code=""" diff --git a/quantdsl/tests/test_readme.py b/quantdsl/tests/test_readme.py index 90e70b8..0fd999d 100644 --- a/quantdsl/tests/test_readme.py +++ b/quantdsl/tests/test_readme.py @@ -1,14 +1,13 @@ import os import sys -from os.path import join, dirname -from subprocess import Popen, PIPE +from os.path import dirname, join +from subprocess import PIPE, Popen from unittest.case import TestCase import quantdsl class TestReadmeFile(TestCase): - def test_code_snippets_in_readme_file(self): # Extract lines of Python code from the README.md file. readme_filename = 'README.md' @@ -45,7 +44,7 @@ def test_code_snippets_in_readme_file(self): readme_py.writelines("\n".join(lines) + '\n') # Run the code and catch errors. - p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_PLOT': 'True'}) + p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_PLOTT': 'True'}) out, err = p.communicate() out = out.decode('utf8').replace(temp_filename, readme_filename) err = err.decode('utf8').replace(temp_filename, readme_filename) From e8e3f58eeca79cc17665648310daebe467797c39 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:12:54 +0100 Subject: [PATCH 46/52] Adjusted README. --- quantdsl/tests/test_readme.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quantdsl/tests/test_readme.py b/quantdsl/tests/test_readme.py index 0fd999d..3e43c82 100644 --- a/quantdsl/tests/test_readme.py +++ b/quantdsl/tests/test_readme.py @@ -44,7 +44,7 @@ def test_code_snippets_in_readme_file(self): readme_py.writelines("\n".join(lines) + '\n') # Run the code and catch errors. - p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_PLOTT': 'True'}) + p = Popen([sys.executable, temp_path], stdout=PIPE, stderr=PIPE, env={'SUPRESS_PLOT': 'True'}) out, err = p.communicate() out = out.decode('utf8').replace(temp_filename, readme_filename) err = err.decode('utf8').replace(temp_filename, readme_filename) From d7ca8121f0f1248406997d21e8344acb394991b4 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:21:49 +0100 Subject: [PATCH 47/52] Adjusted README. --- README.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 93c6bd6..00c7288 100644 --- a/README.md +++ b/README.md @@ -399,9 +399,11 @@ assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ### Power Station -Here's an evaluation of a power station. This time, the source code imports a power station model from the library. +Here's an evaluation of a power station. It uses a market model with two correlated markets. + +This time, the source code imports a power station model from the library. The source code for the power station +model is copied in below. -This example uses a market model with two correlated markets. ```python results = calc_print_plot( @@ -446,3 +448,61 @@ PowerPlant(Date('2012-1-1'), Date('2012-1-6'), Running()) assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ``` + +Quant DSL source code for the library power plant model. + +```python +from quantdsl.semantics import Choice, Lift, Market, TimeDelta, Wait, inline + + +def PowerPlant(start, end, duration_off): + if (start < end): + Wait(start, + Choice( + ProfitFromRunning(duration_off) + PowerPlant( + Tomorrow(start), end, Running() + ), + PowerPlant( + Tomorrow(start), end, Stopped(duration_off) + ) + ) + ) + else: + return 0 + + +@inline +def ProfitFromRunning(duration_off): + if duration_off > 1: + return 0.75 * Power() - Gas() + elif duration_off == 1: + return 0.90 * Power() - Gas() + else: + return 1.00 * Power() - Gas() + + +@inline +def Power(): + Lift('POWER', 'daily', Market('POWER')) + + +@inline +def Gas(): + Lift('GAS', 'daily', Market('GAS')) + + +@inline +def Running(): + return 0 + + +@inline +def Stopped(duration_off): + return duration_off + 1 + + +@inline +def Tomorrow(today): + return today + TimeDelta('1d') + +``` \ No newline at end of file From 3488abe7c16511f7599c08cca86eace6d00ef6ee Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:24:08 +0100 Subject: [PATCH 48/52] Fixed whitespace. --- README.md | 18 +++++++++--------- quantdsl/lib/powerplant2.py | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 00c7288..0997e50 100644 --- a/README.md +++ b/README.md @@ -458,15 +458,15 @@ from quantdsl.semantics import Choice, Lift, Market, TimeDelta, Wait, inline def PowerPlant(start, end, duration_off): if (start < end): Wait(start, - Choice( - ProfitFromRunning(duration_off) + PowerPlant( - Tomorrow(start), end, Running() - ), - PowerPlant( - Tomorrow(start), end, Stopped(duration_off) - ) - ) - ) + Choice( + ProfitFromRunning(duration_off) + PowerPlant( + Tomorrow(start), end, Running() + ), + PowerPlant( + Tomorrow(start), end, Stopped(duration_off) + ) + ) + ) else: return 0 diff --git a/quantdsl/lib/powerplant2.py b/quantdsl/lib/powerplant2.py index b35d82b..b080a4b 100644 --- a/quantdsl/lib/powerplant2.py +++ b/quantdsl/lib/powerplant2.py @@ -4,15 +4,15 @@ def PowerPlant(start, end, duration_off): if (start < end): Wait(start, - Choice( - ProfitFromRunning(duration_off) + PowerPlant( - Tomorrow(start), end, Running() - ), - PowerPlant( - Tomorrow(start), end, Stopped(duration_off) - ) - ) - ) + Choice( + ProfitFromRunning(duration_off) + PowerPlant( + Tomorrow(start), end, Running() + ), + PowerPlant( + Tomorrow(start), end, Stopped(duration_off) + ) + ) + ) else: return 0 From 6569397059f8452c72aabf78b48c3d7959485c6e Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:30:09 +0100 Subject: [PATCH 49/52] Adjusted README. --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0997e50..7f58e78 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,11 @@ has methods that support these steps: `compile()`, `simulate()` and `evaluate()` ## Introduction +The examples below use the library function calc_print_plot() to evaluate contracts. + +```python +from quantdsl.interfaces.calcandplot import calc +``` Simple calculations. @@ -95,12 +100,13 @@ assert results.fair_value == False, results.fair_value ### Function definitions -Function definitions can be used to structure complex expressions. The call args of the function definition -can be used as names in the function definition's expressions. +Function definitions can be used to structure complex expressions. When evaluating an expression that involves calls to function definitions, the call to the function definition is firstly replaced with the expression returned by the function definition, so that a larger expression is formed. +The call args of the function definition can be used as names in the function definition's expressions. The call arg +values will be used to evaluate the expression returned by the function. ```python results = calc(source_code=""" @@ -122,8 +128,8 @@ Contract1(10) assert results.fair_value == 11250, results.fair_value ``` -The function body can be an if-else block, so that the expression returned depends upon the function call argument -values. +The call args of the function definition can be used in an if-else block, so that different expressions can be +returned depending upon the function call argument values. Each call to a (non-inlined) function definition becomes a node on a dependency graph. Each call is internally memoised, so it is only called once with the same argument values, and the result of such a call is reused. From a0d88464c1d72aa8e35bfe8a6fe07bd650be98a6 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:41:02 +0100 Subject: [PATCH 50/52] Adjusted README. --- README.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7f58e78..b3d5dcf 100644 --- a/README.md +++ b/README.md @@ -291,20 +291,25 @@ assert results.fair_value.mean() < 100, results.fair_value.mean() ## Examples of usage -The examples below use the library function `calc_print_plot()` to evaluate contracts. If you run these examples, -the deltas for each market in each period are calculated, and risk neutral hedge positions are printed for each market - in each period, along with the overall fair value. A plot is displayed showing underlying prices, the cumulative hedge positions, and the cummulate cash position from the hedge positions. - The plot shows the statistical distribution of the simulated prices, and the statistical error of the hedge - positions and the cash flow. Comparing the resulting net cash position with the fair value gives an indication of - how well the deltas are performing. +The examples below use the library function `calc_print_plot()` to evaluate contracts. ```python from quantdsl.interfaces.calcandplot import calc_print_plot ``` +If you run these examples, the deltas for each market in each period will be calculated, and risk neutral hedge +positions will be printed for each market in each period, along with the overall fair value. A plot will be +displayed showing underlying prices, the cumulative hedge positions, and the cummulate cash position from the hedge +positions. + +The plot will also show the statistical distribution of the simulated prices, and the statistical error of the hedge + positions and the cash flow. Comparing the resulting net cash position with the fair value gives an indication of + how well the deltas are performing. + + ### Gas Storage -Here's an evaluation of a gas storage facility. +An evaluation of a gas storage facility. This example uses a forward curve that reflects seasonal variations across the term of the contract. @@ -405,7 +410,7 @@ assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ### Power Station -Here's an evaluation of a power station. It uses a market model with two correlated markets. +An evaluation of a power station. It uses a market model with two correlated markets. This time, the source code imports a power station model from the library. The source code for the power station model is copied in below. @@ -455,7 +460,10 @@ assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ``` -Quant DSL source code for the library power plant model. +### Library of models + +Quant DSL source code for the library power plant model `quantdsl.lib.powerplant2`, as used in the example +above. ```python from quantdsl.semantics import Choice, Lift, Market, TimeDelta, Wait, inline From ea4657f7f989afd4feac19136c048e2307423188 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:48:47 +0100 Subject: [PATCH 51/52] Adjusted README. --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b3d5dcf..083e5f4 100644 --- a/README.md +++ b/README.md @@ -291,16 +291,20 @@ assert results.fair_value.mean() < 100, results.fair_value.mean() ## Examples of usage -The examples below use the library function `calc_print_plot()` to evaluate contracts. +The examples below use the library function `calc_print_plot()` to evaluate contracts, and print and plot results. ```python from quantdsl.interfaces.calcandplot import calc_print_plot ``` -If you run these examples, the deltas for each market in each period will be calculated, and risk neutral hedge -positions will be printed for each market in each period, along with the overall fair value. A plot will be -displayed showing underlying prices, the cumulative hedge positions, and the cummulate cash position from the hedge -positions. +### Lift + +The examples here use the `Lift` element to generate deltas with respect each period for each market. + +If you run these examples, the deltas for each market in each period will be calculated, and estimated risk neutral + hedge positions will be printed for each market in each period, along with the overall fair value. A plot will be + displayed showing underlying prices, the cumulative hedge positions, and the cummulate cash position from the hedge + positions. The plot will also show the statistical distribution of the simulated prices, and the statistical error of the hedge positions and the cash flow. Comparing the resulting net cash position with the fair value gives an indication of @@ -309,9 +313,8 @@ The plot will also show the statistical distribution of the simulated prices, an ### Gas Storage -An evaluation of a gas storage facility. - -This example uses a forward curve that reflects seasonal variations across the term of the contract. +An evaluation of a gas storage facility. This example uses a forward curve that reflects seasonal variations across +the term of the contract. ```python results = calc_print_plot( @@ -410,11 +413,8 @@ assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ### Power Station -An evaluation of a power station. It uses a market model with two correlated markets. - -This time, the source code imports a power station model from the library. The source code for the power station -model is copied in below. - +An evaluation of a power station. This time, the source code imports a power station model from the library. It uses + a market model with two correlated markets. The source code for the power station model is copied in below. ```python results = calc_print_plot( @@ -456,8 +456,8 @@ PowerPlant(Date('2012-1-1'), Date('2012-1-6'), Running()) } } ) -assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() +assert 8 < results.fair_value.mean() < 10, results.fair_value.mean() ``` ### Library of models @@ -519,4 +519,4 @@ def Stopped(duration_off): def Tomorrow(today): return today + TimeDelta('1d') -``` \ No newline at end of file +``` From 4d6dda99ec5a62ad892c1ed59dab3bd514320788 Mon Sep 17 00:00:00 2001 From: John Bywater Date: Wed, 20 Sep 2017 04:53:34 +0100 Subject: [PATCH 52/52] Adjusted README. --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 083e5f4..b4cec68 100644 --- a/README.md +++ b/README.md @@ -297,10 +297,6 @@ The examples below use the library function `calc_print_plot()` to evaluate cont from quantdsl.interfaces.calcandplot import calc_print_plot ``` -### Lift - -The examples here use the `Lift` element to generate deltas with respect each period for each market. - If you run these examples, the deltas for each market in each period will be calculated, and estimated risk neutral hedge positions will be printed for each market in each period, along with the overall fair value. A plot will be displayed showing underlying prices, the cumulative hedge positions, and the cummulate cash position from the hedge @@ -311,6 +307,12 @@ The plot will also show the statistical distribution of the simulated prices, an how well the deltas are performing. +### Lift + +The examples here use the `Lift` element to specify deltas with respect to each period (daily, +monthly, yearly) for each market across the term of the contract. + + ### Gas Storage An evaluation of a gas storage facility. This example uses a forward curve that reflects seasonal variations across