Permalink
Switch branches/tags
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
1729 lines (1348 sloc) 63 KB
from __future__ import absolute_import
from __future__ import print_function
import sys
import glob
import time
import numpy as np
np.set_printoptions(threshold=np.nan)
import pandas as pd
import os.path
import time
import datetime
import re
from keras import backend as K
from keras.preprocessing import sequence
from keras.optimizers import SGD, RMSprop, Adagrad
from keras.utils import np_utils
from keras.models import Sequential, Graph, Model
from keras.models import model_from_json
from keras.layers import Input, merge, Flatten, Dense, Activation, Convolution1D, ZeroPadding1D
from keras.layers import TimeDistributed, Reshape
from keras.layers.recurrent import LSTM
from keras.callbacks import TensorBoard
try:
from convolutional_transpose import Convolution1D_Transpose_Arbitrary
except ImportError:
print ("convolutional_transpose.py not available. This might lead to later errors. Are you running Theano?")
#from keras.layers.core import Dense, Dropout, Activation, TimeDistributedDense, Flatten, Reshape, Permute, Merge, Lambda
#from keras.layers.convolutional import Convolution1D, MaxPooling1D, Convolution2D, MaxPooling2D, UpSampling1D, UpSampling2D, ZeroPadding1D
from keras.layers.advanced_activations import ParametricSoftplus, SReLU
from keras.callbacks import ModelCheckpoint, Callback
import matplotlib.pyplot as plt
path = "./training_data_large/" # to make sure signal files are written in same directory as data files
def draw_model(model):
from IPython.display import SVG
from keras.utils.visualize_util import model_to_dot
from keras.utils.visualize_util import plot
#graph = to_graph(model, show_shape=True)
#graph.write_png("UFCNN_1.png")
SVG(model_to_dot(model).create(prog='dot', format='svg'))
plot(model, to_file='UFCNN_1.png')
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
def print_nodes_shapes(model):
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
print(model.summary())
# for k, v in model.inputs.items():
# print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
# for k, v in model.nodes.items():
# print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
# for k, v in model.outputs.items():
# print("{} : {} : {} : {}".format(k, type(v), v.input_shape, v.output_shape))
#
#
def print_layers_shapes(model):
# DEPRECATED DUE TO INTRODUCTION OF model.summary()
print(model.summary())
# for l in model.layers:
# print("{} : {} : {}".format(type(l), l.input_shape, l.output_shape))
def save_neuralnet (model, model_name):
locpath="./"
json_string = model.to_json()
open(locpath + model_name + '_architecture.json', 'w').write(json_string)
model.save_weights(locpath + model_name + '_weights.h5', overwrite=True)
yaml_string = model.to_yaml()
with open(locpath + model_name + '_data.yml', 'w') as outfile:
outfile.write( yaml_string)
def load_neuralnet(model_name):
"""
reading the model from disk - including all the trained weights and the complete model design (hyperparams, planes,..)
"""
locpath="./"
arch_name = locpath + model_name + '_architecture.json'
weight_name = locpath + model_name + '_weights.h5'
if not os.path.isfile(arch_name) or not os.path.isfile(weight_name):
print("model_name given and file %s and/or %s not existing. Aborting." % (arch_name, weight_name))
sys.exit()
print("Loaded model: ",model_name)
try:
model = model_from_json(open(arch_name).read(),{'Convolution1D_Transpose_Arbitrary':Convolution1D_Transpose_Arbitrary})
except NameError: # are we on Theano?
model = model_from_json(open(arch_name).read())
model.load_weights(weight_name)
return model
def ufcnn_model_lstm(sequence_length=5000,
features=1,
nb_filter=150,
filter_length=5,
output_dim=1,
optimizer='adagrad',
loss='mse',
regression = True,
class_mode=None,
activation="softplus",
batch_size = 1,
init="lecun_uniform"):
model = Sequential()
model.add(LSTM(features, stateful=False, return_sequences=True, input_length = features, batch_input_shape=(None, sequence_length, features)))
model.add(TimeDistributed(Dense(5*features)))
#model.add(LSTM(sequence_length, stateful=True, return_sequences=False, batch_input_shape=(1, sequence_length, features)))
#model.add(Dense(sequence_length, activation='relu'))
#model.add(RepeatVector(maxlen))
model.add(LSTM(features, stateful=False, return_sequences=True))
model.add(TimeDistributed(Dense(5*features)))
model.add(LSTM(features, stateful=False, return_sequences=False))
#model.add(Reshape((features*sequence_length,)))
model.add(Dense(output_dim))
model.add(Activation('softmax'))
print(model.summary())
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy', ])
return model
def ufcnn_model_concat(sequence_length=5000,
features=1,
nb_filter=150,
filter_length=5,
output_dim=1,
optimizer='adagrad',
loss='mse',
batch_size = 512,
regression = True,
class_mode=None,
activation="softplus",
init="lecun_uniform"):
#model = Graph()
#model.add_input(name='input', input_shape=(None, features))
main_input = Input(name='input', shape=(sequence_length, features))
#########################################################
#model.add_node(ZeroPadding1D(2), name='input_padding', input='input') # to avoid lookahead bias
input_padding = ZeroPadding1D(2)(main_input) # to avoid lookahead bias
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init, input_shape=(sequence_length, features)), name='conv1', input='input_padding')
#model.add_node(Activation(activation), name='relu1', input='conv1')
conv1 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init)(input_padding)
relu1 = Activation(activation)(conv1)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv2', input='relu1')
#model.add_node(Activation(activation), name='relu2', input='conv2')
conv2 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu1)
relu2 = Activation(activation)(conv2)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv3', input='relu2')
#model.add_node(Activation(activation), name='relu3', input='conv3')
conv3 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu2)
relu3 = Activation(activation)(conv3)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv4', input='relu3')
#model.add_node(Activation(activation), name='relu4', input='conv4')
conv4 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu3)
relu4 = Activation(activation)(conv4)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init), name='conv5', input='relu4')
#model.add_node(Activation(activation), name='relu5', input='conv5')
conv5 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(relu4)
relu5 = Activation(activation)(conv5)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
# name='conv6',
# inputs=['relu3', 'relu5'],
# merge_mode='concat', concat_axis=-1)
#model.add_node(Activation(activation), name='relu6', input='conv6')
merge6 = merge([relu3, relu5], mode='concat')
conv6 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge6)
relu6 = Activation(activation)(conv6)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
# name='conv7',
# inputs=['relu2', 'relu6'],
# merge_mode='concat', concat_axis=-1)
#model.add_node(Activation(activation), name='relu7', input='conv7')
merge7 = merge([relu2, relu6], mode='concat')
conv7 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge7)
relu7 = Activation(activation)(conv7)
#########################################################
#model.add_node(Convolution1D(nb_filter=nb_filter,filter_length=filter_length, border_mode='same', init=init),
# name='conv8',
# inputs=['relu1', 'relu7'],
# merge_mode='concat', concat_axis=-1)
#model.add_node(Activation(activation), name='relu8', input='conv8')
merge8 = merge([relu1, relu7], mode='concat')
conv8 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', init=init)(merge8)
relu8 = Activation(activation)(conv8)
#########################################################
if regression:
#########################################################
#model.add_node(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init), name='conv9', input='relu8')
#model.add_output(name='output', input='conv9')
conv9 = Convolution1D(nb_filter=output_dim, filter_length=filter_length, border_mode='same', init=init)(relu8)
output = conv9
#main_output = conv9.output
else:
#model.add_node(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init), name='conv9', input='relu8')
#model.add_node(Activation('softmax'), name='activation', input='conv9')
#model.add_output(name='output', input='activation')
conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='valid', init=init)(relu8)
activation = (Activation('softmax'))(conv9)
flat = Flatten () (activation)
output = flat
#model.compile(optimizer=optimizer, loss={'output': loss})
model = Model(input=main_input, output=output)
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy', ])
print(model.summary())
return model
def ufcnn_model_deconv(sequence_length=5000,
features=4,
nb_filter=150,
filter_length=5,
output_dim=1,
optimizer='adagrad',
loss='mse',
regression = False,
class_mode=None,
activation="softplus",
init="lecun_uniform"):
strides = [1, 2, 1]
main_input = Input(name='input', shape=(None, features))
#########################################################
# input_padding = ZeroPadding1D(2, name='padding')(main_input) # NOTE: removed in order to keep dimension, re-check. to avoid lookahead bias
#########################################################
conv1 = Convolution1D_Transpose_Arbitrary(nb_filter=nb_filter, filter_length=filter_length, padding='same', strides=strides, init=init, name='conv1')(main_input)
relu1 = Activation(activation, name='relu1')(conv1)
#########################################################
conv2 = Convolution1D_Transpose_Arbitrary(nb_filter=nb_filter, filter_length=filter_length, padding='same', strides=strides, init=init, name='conv2')(relu1)
relu2 = Activation(activation, name='relu2')(conv2)
#########################################################
conv3 = Convolution1D_Transpose_Arbitrary(nb_filter=nb_filter, filter_length=filter_length, padding='same', strides=strides, init=init, name='conv3')(relu2)
relu3 = Activation(activation, name='relu3')(conv3)
#########################################################
conv4 = Convolution1D_Transpose_Arbitrary(nb_filter=nb_filter, filter_length=filter_length, padding='same', strides=strides, init=init, name='conv4')(relu3)
relu4 = Activation(activation, name='relu4')(conv4)
#########################################################
conv5 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', subsample_length=2, init=init, name='conv5')(relu4)
relu5 = Activation(activation, name='relu5')(conv5)
#########################################################
merge6 = merge([relu3, relu5], mode='concat', name='merge6')
conv6 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', subsample_length=2, init=init, name='conv6')(merge6)
relu6 = Activation(activation, name='relu6')(conv6)
#########################################################
merge7 = merge([relu2, relu6], mode='concat', name='merge7')
conv7 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', subsample_length=2, init=init, name='conv7')(merge7)
relu7 = Activation(activation, name='relu7')(conv7)
#########################################################
merge8 = merge([relu1, relu7], mode='concat', name='merge8')
conv8 = Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='same', subsample_length=2, init=init, name='conv8')(merge8)
relu8 = Activation(activation, name='relu8')(conv8)
#########################################################
if regression:
#########################################################
conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init, name='conv9')(relu8)
output = conv9
else:
conv9 = Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init, name='conv9')(relu8)
activation = Activation('softmax', name='activation')(conv9)
output = activation
#########################################################
model = Model(input=main_input, output=output)
model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy', ])
print(model.summary())
return model
def ufcnn_model_seq(sequence_length=5000,
features=1,
nb_filter=150,
filter_length=5,
output_dim=1,
optimizer='adagrad',
loss='mse',
regression = True,
class_mode=None,
init="lecun_uniform"):
model = Sequential()
model.add(ZeroPadding1D(2, input_shape=(None, features)))
#########################################################
model.add(Convolution1D(nb_filter=nb_filter, filter_length=filter_length, border_mode='valid', init=init))
model.add(Activation('relu'))
model.add(Convolution1D(nb_filter=output_dim, filter_length=sequence_length, border_mode='same', init=init))
model.add(Activation('sigmoid'))
model.compile(optimizer=optimizer, loss=loss)
return model
def ufcnn_model(sequence_length=5000,
features=1,
nb_filter=150,
filter_length=5,
output_dim=1,
optimizer='adagrad',
loss='mse',
regression = True,
class_mode=None,
init="lecun_uniform",
mode='concat'):
if mode == 'concat':
return ufcnn_model_concat(sequence_length,
features,
nb_filter,
filter_length,
output_dim,
optimizer,
loss,
regression,
class_mode,
init)
else:
raise NotImplemented
def gen_cosine_amp(amp=100, period=25, x0=0, xn=50000, step=1, k=0.0001):
"""Generates an absolute cosine time series with the amplitude
exponentially decreasing
Arguments:
amp: amplitude of the cosine function
period: period of the cosine function
x0: initial x of the time series
xn: final x of the time series
step: step of the time series discretization
k: exponential rate
Ernst 20160301 from https://github.com/fchollet/keras/blob/master/examples/stateful_lstm.py
as a first test for the ufcnn
"""
cos = np.zeros(((xn - x0) * step, 1, 1))
print("Cos. Shape",cos.shape)
for i in range(len(cos)):
idx = x0 + i * step
cos[i, 0, 0] = amp * np.cos(idx / (2 * np.pi * period))
cos[i, 0, 0] = cos[i, 0, 0] * np.exp(-k * idx)
return cos
def train_and_predict_regression(model, sequence_length=5000, batch_size=128, epochs=5):
lahead = 1
cos = gen_cosine_amp(xn = sequence_length * 100)
expected_output = np.zeros((len(cos), 1, 1))
for i in range(len(cos) - lahead):
expected_output[i, 0] = np.mean(cos[i + 1:i + lahead + 1])
print('Training')
for i in range(epochs):
print('Epoch', i, '/', epochs)
model.fit({'input': cos, 'output': expected_output},
verbose=1,
nb_epoch=1,
shuffle=False,
batch_size=batch_size)
print('Predicting')
predicted_output = model.predict({'input': cos,}, batch_size=batch_size)
return {'model': model, 'predicted_output': predicted_output, 'expected_output': expected_output}
def treat_X_tradcom(mean):
""" treat some columns of the dataframe together when normalizing the dataframe:
col. 1, 2, 4 ... Mkt Price, Bid price, Ask Price
col 3 and 5 ... Ask & Bid price
"""
result = mean.copy()
#print("Result before max",result)
mkt = mean[1]
bid_px = mean[2]
ask_px = mean[4]
px_max=max(mkt,bid_px,ask_px)
result[1] = px_max
result[2] = px_max
result[4] = px_max
bid = mean[3]
ask = mean[5]
ba_max=max(bid,ask)
result[3] = ba_max
result[5] = ba_max
print("Result after max",result)
return result
def standardize_inputs(source, colgroups=None, mean=None, std=None):
"""
Standardize input features.
Groups of features could be listed in order to be standardized together.
source: Pandas.DataFrame or filename of csv file with features
colgroups: list of lists of groups of features to be standardized together (e.g. bid/ask price, bid/ask size)
returns Xdf ...Pandas.DataFrame, mean ...Pandas.DataFrame, std ...Pandas.DataFrame
"""
import itertools
import types
#if isinstance(source, types.StringTypes):
if isinstance(source, str):
Xdf = pd.read_csv(source, sep=" ", index_col = 0, header = None)
elif isinstance(source, pd.DataFrame):
Xdf = source
else:
raise TypeError
df = pd.DataFrame()
me = pd.DataFrame()
st = pd.DataFrame()
for colgroup in colgroups:
_df,_me,_st = standardize_columns(Xdf[colgroup])
# if mean & std are given, do not multiply with colgroup mean
if mean is not None and std is not None:
_df = Xdf[colgroup]
df = pd.concat([df, _df], axis=1)
me = pd.concat([me, _me])
st = pd.concat([st, _st])
print("In Group me")
print(me)
# _temp_list = list(itertools.chain.from_iterable(colgroups))
separate_features = [col for col in Xdf.columns if col not in list(itertools.chain.from_iterable(colgroups))]
if mean is None and std is None:
_me = Xdf[separate_features].mean()
_df = Xdf[separate_features].sub(_me)
_st = Xdf[separate_features].std()
_df = _df[separate_features].div(_st)
else:
_df = Xdf[separate_features]
df = pd.concat([df, _df], axis=1)
me = pd.concat([me, _me])
st = pd.concat([st, _st])
me = pd.Series(me[0])
st = pd.Series(st[0])
if mean is not None and std is not None:
df = df.sub(mean)
df = df.div(std)
return df, me, st
def standardize_columns(colgroup):
"""
Standardize group of columns together
colgroup: Pandas.DataFrame
returns: Pandas.DataFrames: Colum Group standardized, Mean of the colgroup, stddeviation of the colgroup
"""
_me = np.mean(colgroup.values.flatten())
centered = colgroup.sub(_me)
me = pd.DataFrame(np.full(len(colgroup.columns),_me), index=colgroup.columns)
_st = np.std(colgroup.values.flatten())
standardized = centered.div(_st)
st = pd.DataFrame(np.full(len(colgroup.columns),_st), index=colgroup.columns)
return standardized, me, st
def get_tradcom_normalization(filename, mean=None, std=None):
""" read in all X Data Frames and find mean and std of all columns...
"""
Xdf = pd.read_csv(filename, sep=" ", index_col = 0, header = None)
meanLoc = treat_X_tradcom(Xdf.mean())
print("Mean Loc")
print (meanLoc)
sys.stdout.flush()
if mean is None:
mean = meanLoc
mean = mean.to_frame().transpose()
meanDf=pd.concat([mean, meanLoc.to_frame().transpose()])
mean = meanDf.max()
print("Mean")
print (mean)
sys.stdout.flush()
stdLoc = treat_X_tradcom(Xdf.std())
print("Std Loc")
print (stdLoc)
sys.stdout.flush()
if std is None:
std = stdLoc
std = std.to_frame().transpose()
stdDf=pd.concat([std, stdLoc.to_frame().transpose()])
std = stdDf.max()
print("Std")
print (std)
sys.stdout.flush()
return(mean, std)
def prepare_tradcom_classification(training=True,
ret_type='df',
sequence_length=5000,
features_list=[1,2,3,4],
output_dim=3,
file_list=None,
mean=None,
std=None,
training_count=None):
"""
prepare the datasets for the trading competition. training determines which datasets will be read
returns: X and y: Pandas.DataFrames or np-Arrays storing the X - and y values for the fitting.
TODO: refactor - move file operations to separate functions, move stacking to function,
remove commented blocks and undesired print statements
"""
load_file = {'df': pd.read_pickle,
'stack': np.load,
'flat': np.load}
save_file = {'df': lambda filename, obj: obj.to_pickle(filename),
'stack': lambda filename, obj: np.save(filename, obj),
'flat': lambda filename, obj: np.save(filename, obj)}
print("Features_list",features_list)
Xdf = pd.DataFrame()
ydf = pd.DataFrame()
outfile = "training_data_large/save_"+str(len(file_list))
if training:
outfile += "_train"
else:
if training_count is None:
print("Training count needs to be given for testing")
raise ValueError
if mean is None or std is None:
print("Mean & std to be given for testing")
raise ValueError
outfile += "_"+str(training_count)+"_test"
filetype = '.pickle' if ret_type == 'df' else '.npy'
outfile_X = outfile+"_X" + filetype
outfile_y = outfile+"_y" + filetype
outfile_m = outfile+"_m" + filetype
outfile_s = outfile+"_s" + filetype
if os.path.isfile(outfile_X) and os.path.isfile(outfile_y):
X = load_file[ret_type](outfile_X)
y = load_file[ret_type](outfile_y)
#X = np.load(outfile_X)
#y = np.load(outfile_y)
if training:
mean = pd.Series(np.load(outfile_m))
std = pd.Series(np.load(outfile_s))
print("Found files ", outfile_X , " and ", outfile_y)
return (X,y,mean,std)
for filename in file_list:
signalfile = filename.replace('prod_data','signal')
signalfile = signalfile.replace('txt','csv')
print("Working on Input files: ",filename, ", ",signalfile)
if not os.path.isfile(signalfile):
print("File ",signalfile," is not existing. Aborting.")
sys.exit()
# get the date...
r = re.compile('^\D*(\d*)\D*', re.UNICODE)
date = re.search(r, filename).group(1)
print("Date is ",date)
date_ux = time.mktime(datetime.datetime.strptime(date,"%Y%m%d").timetuple())
# load dataframes and reindex
Xdf_loc = pd.read_csv(filename, sep=" ", header = None,)
# print(Xdf_loc.iloc[:3])
Xdf_loc['Milliseconds'] = Xdf_loc[0]
Xdf_loc['Date'] = pd.to_datetime(date_ux*1000*1000*1000)
# Xdf_loc[0] = pd.to_datetime(date_ux*1000*1000*1000 + Xdf_loc[0]*1000*1000)
# Xdf_loc = Xdf_loc.set_index([0])
Xdf_loc = Xdf_loc.set_index(['Date', 'Milliseconds'], append=False, drop=True)
# print(Xdf_loc.iloc[:3])
Xdf = pd.concat([Xdf, Xdf_loc])
print(Xdf.index[0])
print(Xdf.index[-1])
ydf_loc = pd.read_csv(signalfile, names = ['Milliseconds','signal',], )
# print(ydf_loc.iloc[:3])
#ydf_loc['Milliseconds'] = ydf_loc[0]
ydf_loc['Date'] = pd.to_datetime(date_ux*1000*1000*1000)
#ydf_loc[0] = pd.to_datetime(date_ux*1000*1000*1000 + ydf_loc[0]*1000*1000)
#ydf_loc = ydf_loc.set_index([0])
ydf_loc = ydf_loc.set_index(['Date', 'Milliseconds'], append=False, drop=True)
# print(Xdf_loc.iloc[:3])
ydf = pd.concat([ydf, ydf_loc])
#select by features_list
Xdf = Xdf[features_list]
# print("XDF After")
# print(Xdf)
Xdf, mean, std = standardize_inputs(Xdf, colgroups=[[2, 4], [3, 5]], mean=mean, std=std)
# Xdf, mean, std = standardize_inputs(Xdf, colgroups=[[0, 1], ], mean=mean, std=std)
# if nothing from above, the use the calculated data
print("X-Dataframe after standardization")
print(Xdf)
print("Input check")
print("Mean (should be 0)")
print (Xdf.mean())
print("Variance (should be 1)")
print (Xdf.std())
Xdf_array = Xdf.values
X_xdim, X_ydim = Xdf_array.shape
if ret_type == 'stack':
#start_time = time.time()
X = np.zeros((Xdf.shape[0]-sequence_length+1, sequence_length, len(features_list)))
for i in range(0, Xdf.shape[0]-sequence_length+1):
slice = Xdf.values[i:i+sequence_length]
X[i] = slice
#print("Time for Array Fill ", time.time()-start_time)
print(X.shape)
elif ret_type == 'flat':
X = Xdf_array.reshape((1, Xdf_array.shape[0], Xdf_array.shape[1]))
elif ret_type == 'df':
X = Xdf
else:
raise ValueError
#print(X[-1])
#print(_X[-1])
# print(Xdf.iloc[-5:])
ydf['sell'] = ydf.apply(lambda row: (1 if row['signal'] < -0.9 else 0 ), axis=1)
ydf['buy'] = ydf.apply(lambda row: (1 if row['signal'] > 0.9 else 0 ), axis=1)
ydf['hold'] = ydf.apply(lambda row: (1 if row['buy'] < 0.9 and row['sell'] < 0.9 else 0 ), axis=1)
del ydf['signal']
print("Buy signals:", ydf[ydf['buy'] !=0 ].shape[0])
print("Sell signals:", ydf[ydf['sell'] !=0 ].shape[0])
print("% of activity signals", float((ydf[ydf['buy'] !=0 ].shape[0] + ydf[ydf['sell'] !=0 ].shape[0])/ydf.shape[0]))
if ret_type == 'stack':
y = np.zeros((ydf.shape[0]-sequence_length+1, sequence_length, output_dim))
for i in range(0, ydf.shape[0]-sequence_length+1):
slice = ydf.values[i:i+sequence_length]
y[i] = slice
print(y.shape)
elif ret_type == 'flat':
y = ydf.values
y = y.reshape((1, y.shape[0], y.shape[1]))
elif ret_type == 'df':
y = ydf
else:
raise ValueError
save_file[ret_type](outfile_X, X)
save_file[ret_type](outfile_y, y)
# np.save(outfile_X, X)
# np.save(outfile_y, y)
save_file[ret_type](outfile_m, mean)
save_file[ret_type](outfile_s, std)
#np.save(outfile_m, m)
#np.save(outfile_s, s)
return (X,y,mean,std)
def generator(X, y):
print("Call to generator")
print(X.index.equals(y.index))
c = 1
#dates = X.index.get_level_values(0).unique()
while True:
for date_idx in X.index.get_level_values(0).unique():
#print(date_idx)
#print(X.loc[date_idx].shape)
#print(y.loc[date_idx].shape)
X_array = X.loc[date_idx].values
y_array = y.loc[date_idx].values
X_samples = X_array.reshape((1, X_array.shape[0], X_array.shape[1]))
y_samples = y_array.reshape((1, y_array.shape[0], y_array.shape[1]))
#yield {'input': X_samples, 'output': y_samples}
yield (X_samples, y_samples)
def get_lstm_samples(Xdf, ydf, sequence_length = 5000, batch_size = 512):
batch_hide_count = int(sequence_length/batch_size + 1)
nsamples = (int(len(Xdf.index) / batch_size ) - batch_hide_count ) * batch_size
print("Nr. of samples in Generator ")
print("sequence_length: ",sequence_length)
print("batch_size: ",batch_size)
print("batch_hide_count: ",batch_hide_count)
print("nsamples: ",nsamples)
print("len(Xdf.index): ",len(Xdf.index))
return (nsamples, batch_hide_count)
def lstm_generator(Xdf, ydf, sequence_length = 5000, batch_size = 512):
X = np.zeros((sequence_length, len(Xdf.columns)))
X_ret = np.zeros((batch_size, sequence_length, len(Xdf.columns)),dtype = np.float32)
y_ret = np.zeros((batch_size, ydf.shape[1]),dtype = np.float32)
batch_hide_count = int(sequence_length/batch_size + 1)
while 1:
batch_count = 0
batchid = 0
for xx,yy in zip(Xdf.itertuples(), ydf.itertuples()):
X = np.roll(X, -1, axis=0)
X[sequence_length-1] = np.array(xx[1:])
##print("\nYYTUPLE ",yy, xx)
if batch_count >= batch_hide_count: #??
y_ret[batchid] = np.array(yy[1:])
##print ("\nARGMAXY",np.argmax(np.array(yy[1:])))
X_ret[batchid] = np.copy(X)
#print("GENERATO")
#print(X_ret)
#ret = y.reshape((1,-1))
#yield (X.reshape((1,X.shape[0],X.shape[1])), ret)
batchid += 1
if batchid == batch_size:
batchid = 0
batch_count += 1
if batch_count > batch_hide_count: #??
#print ("\nGENERATURN X_ret: ",batch_count,",",batch_size,"\n")
#print(X_ret)
#print("\nGENERATURN y_ret: ",batch_count,"\n")
#print(y_ret)
print ("Yielding batchcount", batch_count)
yield (X_ret, y_ret)
def train_and_predict_classification(model, sequence_length=5000, features=32, output_dim=3, batch_size=128, epochs=5, name = "model", training_count=3, testing_count=3):
final_loss = 0
file_list = sorted(glob.glob('./training_data_large/prod_data_*v.txt'))
if len(file_list) == 0:
print ("Files ./training_data_large/product_data_*txt and signal_*.csv are needed. Please copy them in the ./training_data_large/ . Aborting.")
sys.exit()
line = []
mean = None
std = None
for j in range(training_count):
filename = file_list[j]
print('Normalizing: ',filename)
# (mean, std) = get_tradcom_normalization(filename = filename, mean = mean, std = std)
# here i removed some intendation
for j in range(training_count):
filename = file_list[j]
print('Training: ',filename)
X,y = prepare_tradcom_classification(training = True, sequence_length = sequence_length, features = features, output_dim = output_dim, filename = filename, mean = mean, std = std)
# running over all epochs to get the optimizer working well...
history = model.fit({'input': X, 'output': y},
verbose=1,
nb_epoch=epochs,
shuffle=False,
batch_size=batch_size)
print(history.history)
sys.stdout.flush()
final_loss = history.history['loss']
line.extend(final_loss)
save_neuralnet (model, "ufcnn_"+str(j))
plt.figure()
plt.plot(line)
plt.savefig("Convergence.png")
#plt.show()
total_class_count = 0
total_correct_class_count = 0
for k in range(testing_count):
filename = file_list[training_count + k]
print("Predicting: ",filename)
X,y,mean,std = prepare_tradcom_classification(training=False, sequence_length=sequence_length, features=features, output_dim=output_dim, filename=filename, mean=mean, std=std )
predicted_output = model.predict(X, batch_size=batch_size, verbose = 2)
#print(predicted_output)
yp = predicted_output['output']
xdim, ydim = yp.shape
## MSE for testing
total_error = 0
correct_class= 0
for i in range (xdim):
delta = 0.
for j in range(ydim):
delta += (y[i][j] - yp[i][j]) * (y[i][j] - yp[i][j])
#print ("Row %d, MSError: %8.5f " % (i, delta/ydim))
total_error += delta
if np.argmax(y[i]) == np.argmax(yp[i]):
correct_class += 1
print ("FIN Correct Class Assignment: %6d /%7d" % (correct_class, xdim))
print ("FIN Final Loss: ", final_loss)
total_class_count += xdim
total_correct_class_count += correct_class
print ("FINFIN Correct Class Assignment: %6d /%7d" % (total_correct_class_count, total_class_count))
return {'model': model, 'predicted_output': predicted_output['output'], 'expected_output': y}
#def check_prediction(Xdf, y, yp, mean, std):
# """ Check the predicted classes and print results
# """
# ## MSE for testing
# total_error = 0
# correct_class= 0
#
# y_pred_class = np.zeros((y.shape[2],))
# y_corr_pred_class = np.zeros((y.shape[2],))
# y_class = np.zeros((y.shape[2],))
# y_labels = np.zeros((y.shape[1], y.shape[2]))
# a=['Sell','Buy','Hold']
#
# for i in range (y.shape[1]):
# delta = 0.
# for j in range(y.shape[2]):
# delta += (y[0][i][j] - yp[0][i][j]) * (y[0][i][j] - yp[0][i][j])
#
# total_error += delta
#
# #if np.any(y[0][i] != 0): # some debug output, comment if not needed!
# # print("Actual: ", y[0][i])
# # print("Predicted: ", yp[0][i])
#
# if np.argmax(y[0][i]) == np.argmax(yp[0][i]):
# correct_class += 1
# y_corr_pred_class[np.argmax(yp[0][i])] += 1.
#
# y_pred_class[np.argmax(yp[0][i])] += 1.
# y_class[np.argmax(y[0][i])] += 1.
# y_labels[i][np.argmax(yp[0][i])] = 1
#
# print()
# print("Total MSE Error: ", total_error / y.shape[1])
# print("Correct Class Assignment: %6d /%7d" % (correct_class, y.shape[1]))
#
# for i in range(y.shape[2]):
# print("%4s: Correctly Predicted / Predicted / Total: %6d/%6d/%7d" %(a[i], y_corr_pred_class[i], y_pred_class[i], y_class[i]))
#
# Xdf = Xdf * std
# Xdf = Xdf + mean
#
# yp_p = yp.reshape((yp.shape[1],yp.shape[2]))
# #print(yp_p)
#
# ydf2 = pd.DataFrame(yp_p, columns=['sell','buy','hold'])
# Xdf2 = Xdf.reset_index(drop=True)
# Xdf2 = pd.concat([Xdf2,ydf2], axis = 1)
#
# Xdf2['signal'] = 0.
# print(Xdf2)
#
# xy_df = pd.concat([Xdf, pd.DataFrame(y_labels, columns=['sell','buy','hold'], index=Xdf.index)], axis=1)
# xy_df = xy_df.rename(columns={2: "bidpx_", 3: "bidsz_", 4: "askpx_", 5: "asksz_"})
#
#
# # store everything in signal
# # -1 for short, 1 for long...
# Xdf2['signal'] = Xdf2.apply(lambda row: (1 if row['buy'] > row['hold'] and row['buy'] > row['sell'] else 0 ), axis=1)
# Xdf2['signal'] = Xdf2.apply(lambda row: (-1 if row['sell'] > row['hold'] and row['sell'] > row['buy'] else row['signal'] ), axis=1)
#
# invested_tics = 0
# pnl = 0.
# position = 0.
# last_row = None
# nr_trades = 0
# trade_pnl = 0.
#
# for (index, row) in Xdf2.iterrows():
# (pnl_, position, is_trade) = calculate_pnl(position, last_row, row, fee_per_roundtrip=0.0)
# pnl += pnl_
#
# last_row = row
# if position < -0.1 or position > 0.1:
# invested_tics +=1
# if is_trade:
# nr_trades += 1
# trade_pnl = 0.
# trade_pnl += pnl_
#
# sig_pnl, sig_trades = get_pnl(xy_df)
# print("Signals PnL: {}, # of trades: {}".format(sig_pnl, sig_trades))
#
# print ("Nr of trades: %5d /%7d" % (nr_trades, y.shape[1]))
# print ("PnL: %8.2f InvestedTics: %5d /%7d" % (pnl, invested_tics, y.shape[1]))
#
# return pnl
### END
def check_prediction(Xdf, y, yp, mean, std, day=None, offset = 0):
""" Check the predicted classes and print results
results of this version (ufcnn6) where checked by Stefan on 20160424
XData Frame, y...correct y, yp...predicted
"""
## MSE for testing
total_error = 0
correct_class= 0
y_pred_class = np.zeros((y.shape[2],))
y_corr_pred_class = np.zeros((y.shape[2],))
y_class = np.zeros((y.shape[2],))
y_labels = np.zeros((y.shape[1], y.shape[2]))
yp_transf = np.zeros((1, y.shape[1], y.shape[2]))
yp_transf[0,:,2] = 1 # standard hold
a=['Sell','Buy','Hold']
len_y = y.shape[1]
len_yp = yp.shape[1]
len_min = min(len_y, len_yp)
print ("Y Shape ", y.shape)
print ("YP Shape ", yp.shape)
for i in range (yp.shape[1]):
yp_transf[0][i + offset][:] = yp[0][i][:]
print("TRANSF",yp_transf[0][i + offset], yp[0][i])
for i in range (y.shape[1]):
delta = 0
for j in range(y.shape[2]):
delta += (y[0][i][j] - yp_transf[0][i][j]) * (y[0][i][j] - yp_transf[0][i][j])
total_error += delta
#if np.any(y[0][i] != 0): # some debug output, comment if not needed!
# print("Actual: ", y[0][i])
# print("Predicted: ", yp[0][i])
if np.argmax(y[0][i]) == np.argmax(yp_transf[0][i]):
correct_class += 1
y_corr_pred_class[np.argmax(yp_transf[0][i])] += 1.
y_pred_class[np.argmax(yp_transf[0][i])] += 1.
y_class[np.argmax(y[0][i])] += 1.
y_labels[i][np.argmax(yp_transf[0][i])] = 1
print()
print("Total MSE Error: ", total_error / y.shape[1])
print("Correct Class Assignment: %6d /%7d" % (correct_class, y.shape[1]))
for i in range(y.shape[2]):
print("%4s: Correctly Predicted / Predicted / Total: %6d/%6d/%7d" %(a[i], y_corr_pred_class[i], y_pred_class[i], y_class[i]))
Xdf = Xdf * std
Xdf = Xdf + mean
yp_p = yp_transf.reshape((yp_transf.shape[1],yp_transf.shape[2]))
#print(yp_p)
ydf2 = pd.DataFrame(yp_p, columns=['sell','buy','hold'])
Xdf2 = Xdf.reset_index(drop=True)
Xdf2 = pd.concat([Xdf2,ydf2], axis = 1)
Xdf2['signal'] = 0.
print(Xdf2)
xy_df = pd.concat([Xdf, pd.DataFrame(y_labels, columns=['sell','buy','hold'], index=Xdf.index)], axis=1)
xy_df = xy_df.rename(columns={2: "bidpx_", 3: "bidsz_", 4: "askpx_", 5: "asksz_"})
# store everything in signal
# -1 for short, 1 for long...
Xdf2['signal'] = Xdf2.apply(lambda row: (1 if row['buy'] > row['hold'] and row['buy'] > row['sell'] else 0 ), axis=1)
Xdf2['signal'] = Xdf2.apply(lambda row: (-1 if row['sell'] > row['hold'] and row['sell'] > row['buy'] else row['signal'] ), axis=1)
print("Xdf2")
print(Xdf2)
invested_tics = 0
pnl = 0.
position = 0.
last_row = None
nr_trades = 0
trade_pnl = 0.
last_open_rate = None
last_position = 0
open_index = 0
for (index, row) in Xdf2.iterrows():
(pnl_, position, is_trade, close_rate, open_rate, current_pnl) = calculate_pnl(position, last_row, row, fee_per_roundtrip=0.0)
last_row = row
if position < -0.1 or position > 0.1:
invested_tics +=1
if is_trade:
if last_open_rate is not None:
print("TradeData;",nr_trades,";",last_position,";",day,";",open_index,";",last_open_rate,";",index,";",close_rate,";",trade_pnl,";",pnl)
nr_trades += 1
trade_pnl = current_pnl
open_index = index
last_position = position
last_open_rate = open_rate
else:
trade_pnl += pnl_
pnl += pnl_
sig_pnl, sig_trades = get_pnl(xy_df)
print("Signals PnL: {}, # of trades: {}".format(sig_pnl, sig_trades))
print ("Nr of trades: %5d /%7d" % (nr_trades, y.shape[1]))
print ("PnL: %8.2f InvestedTics: %5d /%7d, Nr. of Trades %5d" % (pnl, invested_tics, y.shape[1], nr_trades))
return pnl
### END
def calculate_pnl(position, row, next_row, fee_per_roundtrip=0.):
"""
results of this version (ufcnn6) where checked by Stefan on 20160424
"""
close_rate = 0.
open_rate = 0.
current_pnl = 0
if row is None or next_row is None:
return (0.,0., False, close_rate, open_rate, 0.0)
old_position = position
pnl = 0.
signal = row['signal']
# if we are short and need to go long...
if position < -0.1 and signal > 0.1:
close_rate = row[4]
position = 0.
# if we are long and need to go short...
if position > 0.1 and signal < -0.1:
close_rate = row[2]
position = 0.
last_rate = 0
if position < -0.1:
pnl = position * (next_row[4] - row[4]) # ASK
if position > 0.1:
pnl = position * (next_row[2] - row[2]) # BID
trade = False
if position == 0. and abs(signal) > 0.1:
position = signal
if position < -0.1:
current_pnl = position * (next_row[4] - row[2]) # ASK
open_rate = row[2]
if position > 0.1:
current_pnl = position * (next_row[2] - row[4]) # BID
open_rate = row[4]
current_pnl -= fee_per_roundtrip
trade = True
pnl = pnl + current_pnl
#print ("SIGNAL:",signal, ", old_position: ", old_position, " position:", position, ", pnl: ",pnl, "Bid: ",row[2],next_row[2],", ASK ",row[4], next_row[4] )
return (pnl, position, trade, close_rate, open_rate, current_pnl)
## End calculate_pnl
def get_pnl(df, max_position=1, comission=0):
deals = []
pnl = 0
position = 0
df_with_signals = df[(df['sell'] != 0) | (df['buy'] != 0)]
for idx, row in df_with_signals.iterrows():
if row['buy'] == 1 and position < max_position:
###print(row)
current_trade = -row['buy'] * row["askpx_"]
position += 1
pnl = pnl + current_trade - comission
deals.append(current_trade)
###print("Running PnL: {}, position: {}".format(pnl, position))
elif row['sell'] == 1 and position > -max_position:
###print(row)
current_trade = row['sell'] * row["bidpx_"]
position -= 1
pnl = pnl + current_trade - comission
deals.append(current_trade)
###print("Running PnL: {}, position: {}".format(pnl, position))
if position == 1:
day_closing_trade = df.iloc[-1]["bidpx_"]
pnl = pnl + day_closing_trade - comission
deals.append(day_closing_trade)
###print("Close last hanging deal on the end of the day, PnL: {}, position: {}".format(pnl, position))
elif position == -1:
day_closing_trade = -df.iloc[-1]["askpx_"]
pnl = pnl + day_closing_trade - comission
deals.append(day_closing_trade)
###print("Close last hanging deal on the end of the day, PnL: {}, position: {}".format(pnl, position))
print("Check PnL: {} vs {}".format(pnl, np.sum(deals)))
return pnl, len(deals)
#def calculate_pnl(position, row, next_row, fee_per_roundtrip=0.):
# if row is None:
# return (0.,0., False)
#
# old_position = position
#
# pnl = 0.
#
# if position < -0.1:
# pnl = position * (next_row[4] - row[4]) # ASK
#
# if position > 0.1:
# pnl = position * (next_row[2] - row[2]) # BID
#
#
# signal = row['signal']
#
# # if we are short and need to go long...
# if position < -0.1 and signal > 0.1:
# position = 0.
#
# # if we are long and need to go short...
# if position > 0.1 and signal < -0.1:
# position = 0.
#
# trade = False
# if position == 0. and abs(signal) > 0.1:
# position = signal
# if position < -0.1:
# pnl = position * (next_row[4] - row[2]) # ASK
#
# if position > 0.1:
# pnl = position * (next_row[2] - row[4]) # BID
# pnl -= fee_per_roundtrip
# trade = True
#
# #print ("SIGNAL:",signal, ", old_position: ", old_position, " position:", position, ", pnl: ",pnl, "Bid: ",row[2],next_row[2],", ASK ",row[4], next_row[4] )
# return (pnl, position, trade)
## End calculate_pnl
def get_tracking_data (sequence_length=5000, count=2000, D=10, delta=0.3, omega_w=0.005, omega_ny=0.005):
""" get tracking data for a target moving in a square with 2D side length
delta ... radius of the round target
omega_ ... random noise strength
"""
A = np.array([[1,1,0,0],[0,1,0,0],[0,0,1,1],[0,0,0,1]])
X = np.zeros((count,sequence_length,1))
y = np.zeros((count,sequence_length,2))
for i in range(count):
z_t = np.random.normal(1,.5,4)
g_t = np.random.normal(1,.5,4)
x_t = z_t[0]
xp_t = z_t[1]
y_t = z_t[2]
yp_t = z_t[3]
for j in range(sequence_length):
# reflect at the border of the square with length 2D
if -D + delta < x_t and x_t < D - delta:
xp_new_t = xp_t
elif -D + delta <= x_t:
xp_new_t = -abs(xp_t)
else:
xp_new_t = abs(xp_t)
if -D + delta < y_t and y_t < D - delta:
yp_new_t = yp_t
elif -D + delta <= y_t:
yp_new_t = -abs(yp_t)
else:
yp_new_t = abs(yp_t)
g_t[0] = x_t
g_t[1] = xp_new_t
g_t[2] = y_t
g_t[3] = yp_new_t
w_t = np.random.normal(0.,0.5*omega_w,4)
w_t[1] = 0.
w_t[3] = 0.
ny_t = np.random.normal(0.,0.5*omega_ny,1)
z_t = np.dot(A, g_t) + w_t
x_t = z_t[0]
xp_t = z_t[1]
y_t = z_t[2]
yp_t = z_t[3]
theta = np.arctan(y_t/x_t) + ny_t[0]
# params for the nn
# learn to predict x&y by bearing (theta)
X[i][j][0] = theta
y[i][j][0] = x_t
y[i][j][1] = y_t
#print ("X_T: ", x_t, ", Y_T: ",y_t)
return (X,y)
def get_simulation(write_spans = True):
"""
Make trading competition-like input and output data from the cosine function
"""
from signals import find_all_signals, make_spans, set_positions, pnl
from datetime import date
df = pd.DataFrame(data={"askpx_": np.round(gen_cosine_amp(k=0, period=10, amp=20)[:, 0, 0]+201),
"bidpx_": np.round(gen_cosine_amp(k=0, period=10, amp=20)[:, 0, 0]+200)})
df = find_all_signals(df)
df = make_spans(df, 'Buy')
df = make_spans(df, 'Sell')
print("Simulation PnL", pnl(df))
Xdf = df[["askpx_", "bidpx_"]]
df['buy'] = df['Buy'] if not write_spans else df['Buys']
df['sell'] = df['Sell'] if not write_spans else df['Sells']
ydf = df[["sell", "buy"]]
Xdf['Milliseconds'] = Xdf.index
Xdf['Date'] = pd.to_datetime(date.today())
Xdf = Xdf.set_index(['Date', 'Milliseconds'], append=False, drop=True)
#print(Xdf.index[0:100])
ydf['Milliseconds'] = ydf.index
ydf['Date'] = pd.to_datetime(date.today())
ydf = ydf.set_index(['Date', 'Milliseconds'], append=False, drop=True)
#print(ydf.index[0:100])
Xdf, mean, std = standardize_inputs(Xdf, colgroups=[["askpx_", "bidpx_"], ])
ydf['hold'] = ydf.apply(lambda row: (1 if row['buy'] == 0 and row['sell'] == 0 else 0 ), axis=1)
print("Buy signals:", ydf[ydf['buy'] !=0 ].shape[0])
print("Sell signals:", ydf[ydf['sell'] !=0 ].shape[0])
print("% of activity signals", float(ydf[ydf['buy'] !=0 ].shape[0] + ydf[ydf['sell'] !=0 ].shape[0])/float(ydf.shape[0]))
print(Xdf.shape, Xdf.columns)
print(ydf.shape, ydf.columns)
return (Xdf,ydf,mean,std)
def test_look_ahead(model, sequence_length, X_pred, y_pred):
"""
test whether the model can predict into the future or is reading the future from the data
"""
print ("test_look_ahead:")
#sequence_length = 5000
time_shift = 3 # >= 0
shifts = [3,5,10,15,20,25,40]
for time_shift in shifts:
print ("Timeshift for InvestedTics: ",time_shift)
i=0
for date_idx in X_pred.index.get_level_values(0).unique():
p=0
X_array = X_pred.loc[date_idx].values
y_array = y_pred.loc[date_idx].values
X_samples = X_array.reshape((1, X_array.shape[0], X_array.shape[1]))
y_samples = y_array.reshape((1, y_array.shape[0], y_array.shape[1]))
print("Checking full vs. partial Prediction for day ",i ,": ", date_idx)
# predict over the full X_samples array for comparison
predicted_output_full = model.predict(X_samples, batch_size=1, verbose=2)
predicted_output_upd = predicted_output_full.copy()
print (X_samples.shape)
# feed small squences of data to the model and check wheter the preedictions are identical to predictions done in a
# single run over all data
error_count = 0
total_count = 0
# 0 ... the leftmost tick fed to predict
# loop through the day, ignore the first sequence_length samples (this is a simplification!)
for j in range(sequence_length, X_samples.shape[1]):
total_count += 1
predicted_output = model.predict(X_samples[:,j-sequence_length:j,:], batch_size=1, verbose=2)
print("full:single: ", j, predicted_output_full[0,j-1-time_shift], predicted_output[0,sequence_length-1-time_shift])
# check whether the signals are identical
if np.argmax(predicted_output_full[0,j-1-time_shift]) != np.argmax(predicted_output[0,sequence_length-1-time_shift]):
error_count += 1
print("Error: MaxArg:", np.argmax(predicted_output_full[0,j-1-time_shift]), np.argmax(predicted_output[0,sequence_length-1-time_shift]))
# store the time-shifted prediction
predicted_output_upd[0,j-1-time_shift,:] = predicted_output[0,sequence_length-1-time_shift,:]
if p < 3:
p += 1
np.set_printoptions(threshold=np.nan)
print("First 3 Predictions")
print(predicted_output)
#test the result of the time-shifted prediction
pnl = check_prediction(X_pred.loc[date_idx], y_samples, predicted_output_upd, mean, std)
i+= 1
print (i,"Date: ",date_idx, "Errors/Total: ", error_count, "/", total_count, "New Pnl", pnl)
# only 3 days...
if i > 2:
break
# test_look_ahead() END
def compare_array(hide_count, array1, array2, array3 = None):
array_list = [array1, array2]
if array3 is not None:
array_list.append(array3)
comp_file = open('compare.csv', 'w')
for i in range(hide_count, array1.shape[1]):
for array in array_list:
arg = np.argmax(array[0,i,:])
comp_file.write(str(arg))
comp_file.write(';')
for j in range(3):
comp_file.write("%.4f" % array[0,i,j])
comp_file.write(';')
comp_file.write(';')
comp_file.write("\n")
comp_file.close()
#########################################################
## Test the net with damped cosine / remove later...
#########################################################
if len(sys.argv) < 2 :
print ("Usage: UFCNN1.py action with action from [cos_small, cos, tradcom, tradcom_simple, tracking] [model_name]")
print(" ... with model_name = name of the saved file (without addition like _architecture...) to load the net from file")
sys.exit()
action = sys.argv[1]
if len(sys.argv) == 3:
model_name = sys.argv[2]
else:
model_name = None
sequence_length = 64 # same as in Roni Mittelman's paper - this is 2 times 32 - a line in Ronis input contains 33 numbers, but 1 is time and is omitted
features = 1 # guess changed Ernst 20160301
nb_filter = 150 # same as in Roni Mittelman's paper
filter_length = 5 # same as in Roni Mittelman's paper
output_dim = 1 # guess changed Ernst 20160301
if action == 'cos_small':
print("Running model: ", action)
UFCNN_1 = ufcnn_model(sequence_length=sequence_length)
print_nodes_shapes(UFCNN_1)
case_1 = train_and_predict_regression(UFCNN_1, sequence_length=sequence_length)
print('Ploting Results')
plt.figure(figsize=(18,3))
plt.plot(case_1['expected_output'].reshape(-1)[-10000:]) #, predicted_output['output'].reshape(-1))
plt.plot(case_1['predicted_output']['output'].reshape(-1)[-10000:])
#plt.savefig('sinus.png')
plt.show()
if action == 'cos':
print("Running model: ", action)
UFCNN_2 = ufcnn_model()
print_nodes_shapes(UFCNN_2)
case_2 = train_and_predict_regression(UFCNN_2)
print('Ploting Results')
plt.figure(figsize=(18,3))
plt.plot(case_2['expected_output'].reshape(-1)[-10000:]) #, predicted_output['output'].reshape(-1))
plt.plot(case_2['predicted_output']['output'].reshape(-1)[-10000:])
#plt.savefig('sinus.png')
plt.show()
if action == 'tradcom':
print("Running model: ", action)
sequence_length = 500
features = 4
output_dim = 3
# Roni used rmsprop
sgd = SGD(lr=0.005, decay=1e-6, momentum=0.9, nesterov=True)
UFCNN_TC = ufcnn_model(regression = False, output_dim=output_dim, features=features,
loss="categorical_crossentropy", sequence_length=sequence_length, optimizer=sgd )
#print_nodes_shapes(UFCNN_TC)
case_tc = train_and_predict_classification(UFCNN_TC, features=features, output_dim=output_dim, sequence_length=sequence_length, epochs=50, training_count=10, testing_count = 6 )
if action == 'tracking':
print("Running model: ", action)
sequence_length = 5000
count=20
output_dim = 2
# Roni used rmsprop
sgd = SGD(lr=0.00005, decay=1e-6, momentum=0.9, nesterov=True)
rms = RMSprop(lr=0.001, rho=0.9, epsilon=1e-06)
model = ufcnn_model_concat(regression = True, output_dim=output_dim, features=features,
loss="mse", sequence_length=sequence_length, optimizer=rms )
#print_nodes_shapes(UFCNN_TC)
(X,y) = get_tracking_data (sequence_length=sequence_length, count=count)
X = np.subtract(X,X.mean())
y = np.subtract(y,y.mean())
#plt.figure()
#plt.plot(x1, y1)
#plt.savefig("TrackingTracking.png")
history = model.fit({'input': X, 'output': y},
verbose=1,
nb_epoch=300)
print(history.history)
if action == 'tradcom_simple':
simulation = False # Use True for simulated cosine data, False - for data from files
training_count = 1
validation_count = 1
testing_count = 1
sequence_length = 500
#features_list = list(range(0,2)) # list(range(2,6)) #list(range(1,33))
if not simulation:
features_list = list(range(2,6)) ## to run with Bid/Ask price/vol only
features_list = list(range(1,33)) ## FULL
file_list = sorted(glob.glob('./training_data_large/prod_data_*v.txt'))[:training_count]
print ("Training file list ", file_list)
(X, y, mean, std) = prepare_tradcom_classification(training=True,
ret_type='df',
sequence_length=sequence_length,
features_list=features_list,
output_dim=3,
file_list=file_list)
file_list = sorted(glob.glob('./training_data_large/prod_data_*v.txt'))[training_count:training_count+validation_count]
print ("Validation file list ", file_list)
(X_val, y_val, mean_, std_) = prepare_tradcom_classification(training=True,
ret_type='df',
sequence_length=sequence_length,
features_list=features_list,
output_dim=3,
file_list=file_list,
mean=mean,
std=std,
training_count=training_count)
else:
features_list = list(range(0,2))
print("Using simulated data for training...")
(X, y, mean, std) = get_simulation()
(X_val, y_val, mean_, std_) = get_simulation()
#
print("X ALL shape: ", X.shape)
print(X)
print("Y ALL shape: ", y.shape)
print(y)
#
# print("Mean")
# print(mean)
# print("Std")
# print(std)
#for _d in generator(X, y):
# print(_d)
sgd = SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True)
#rmsprop = RMSprop (lr=0.00001, rho=0.9, epsilon=1e-06) # for sequence length 500
rmsprop = RMSprop (lr=0.000005, rho=0.9, epsilon=1e-06) # for sequence length 5000
# load the model from disk if model name is given...
loss="categorical_crossentropy"
if model_name is not None:
model = load_neuralnet(model_name)
model.compile(optimizer=rmsprop, loss=loss, metrics=['accuracy', ])
else:
model = ufcnn_model_concat(regression = False, output_dim=3, features=len(features_list),
loss=loss, sequence_length=sequence_length, optimizer=rmsprop, batch_size = 320 )
print_nodes_shapes(model)
#draw_model(model)
#history = model.fit({'input': X, 'output': y},
# verbose=2,
# nb_epoch=5,
# shuffle=False,
# batch_size=1)
if K._config['backend'] != 'theano':
callbacks = [TensorBoard(), ]
else:
callbacks = []
start_time = time.time()
epoch = 40
epoch = 0
use_lstm = True
batch_size = 320
(samples_per_epoch,_) = get_lstm_samples(X, y, sequence_length, batch_size)
if True:
history = model.fit_generator(lstm_generator(X, y, sequence_length, batch_size),
samples_per_epoch=samples_per_epoch,
verbose=1,
nb_epoch=epoch,
callbacks=callbacks)
save_neuralnet (model, "ufcnn_sim") if simulation else save_neuralnet (model, "ufcnn_concat_seq")
if epoch > 0:
print("--- Fitting: Elapsed: %d seconds per iteration %5.3f" % ( (time.time() - start_time),(time.time() - start_time)/epoch))
print("1st TEST")
print(history.history)
losstest = model.evaluate_generator(lstm_generator(X, y, sequence_length, batch_size), samples_per_epoch)
print("ORIG LOSSTEST XY: ",losstest)
if not simulation:
# and get the files for testing
file_list = sorted(glob.glob('./training_data_large/prod_data_*v.txt'))[training_count:training_count+testing_count]
(X_pred, y_pred, mean_, std_) = prepare_tradcom_classification(training=False,
ret_type='df',
sequence_length=sequence_length,
features_list=features_list,
output_dim=3,
file_list=file_list,
mean=mean,
std=std,
training_count=training_count)
else:
print("Using simulated data for training...")
(X_pred, y_pred, mean_, std_) = get_simulation()
print(X_pred.iloc[0:200])
print(y_pred.iloc[0:200])
count=0
pnl = 0.
for date_idx in X_pred.index.get_level_values(0).unique():
X_array = X_pred.loc[date_idx].values
y_array = y_pred.loc[date_idx].values
X_samples = X_array.reshape((1, X_array.shape[0], X_array.shape[1]))
y_samples = y_array.reshape((1, y_array.shape[0], y_array.shape[1]))
predicted_output_total=np.zeros((1, y_array.shape[0], y_array.shape[1]))
(samples_per_epoch, hide_count) = get_lstm_samples(X_pred.loc[date_idx], y_pred.loc[date_idx], sequence_length, batch_size)
hide_count *= batch_size
count += 1
y_pred_gen = model.predict_generator(lstm_generator(X_pred.loc[date_idx], y_pred.loc[date_idx], sequence_length, batch_size), samples_per_epoch, max_q_size=1)
y_pred_gen_trf = np.zeros((1,y_samples.shape[1],y_samples.shape[2]))
y_pred_gen_trf[0,:,2] = 1.
for j in range(y_pred_gen.shape[0]):
y_pred_gen_trf[0,hide_count+j,:]=y_pred_gen[j,:]
pnl_= check_prediction(X_pred.loc[date_idx], y_samples, y_pred_gen_trf, mean, std, offset = 0)
pnl += pnl_
print("Predicting: day ",count ,": ", date_idx, 'PnL: ', pnl_)
print ("Average PnL: ", pnl/count)
# Test how big the future bias would be and how big the differences / PnL would be
test_future_bias = False
if test_future_bias:
test_look_ahead(model, sequence_length, X_pred, y_pred)