Skip to content

Commit

Permalink
Start groundwork for K-fold validation
Browse files Browse the repository at this point in the history 
Need to improve the generator and ideally get the streaming from directory to work as well.
  • Loading branch information
@jacobbieker
jacobbieker committed Oct 29, 2018
1 parent 522ee36 commit acb14e7
Show file tree
Hide file tree
Showing 6 changed files with 97 additions and 59 deletions.
10 changes: 5 additions & 5 deletions examples/energy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
'train_fraction': 0.6,
'validate_fraction': 0.2,
'mode': 'train',
'samples': 800000,
'samples': 20000,
'chunked': False,
'augment': True,
}
Expand All @@ -45,21 +45,21 @@
energy_model_configuration = {
'conv_dropout': 0.1,
'lstm_dropout': 0.2,
'fc_dropout': 0.5,
'fc_dropout': 0.3,
'num_conv3d': 3,
'kernel_conv3d': 2,
'strides_conv3d': 1,
'num_lstm': 1,
'kernel_lstm': 2,
'strides_lstm': 1,
'num_fc': 3,
'num_fc': 2,
'pooling': True,
'neurons': [32, 32, 16, 16, 16, 32, 48, 64],
'neurons': [32, 64, 128, 64, 32, 64],
'shape': [25, 38, 38, 1],
'start_slice': 0,
'number_slices': 25,
'activation': 'relu',
'name': 'testEnergy',
'patience': 400,
}

energy_model = EnergyModel(config=energy_model_configuration)
Expand Down
13 changes: 8 additions & 5 deletions examples/source_detection.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from factnn import GammaDiffusePreprocessor, DispGenerator, DispModel, SignGenerator, SignModel
import os.path
import numpy as np

base_dir = "../ihp-pc41.ethz.ch/public/phs/"
obs_dir = [base_dir + "public/"]
Expand All @@ -23,6 +24,10 @@
if not os.path.isfile(gamma_diffuse_configuration["output_file"]):
gamma_diffuse_preprocessor.create_dataset()

# Since the Gamma Diffuse simulations seem to be located in order, this should allow for better testing of small groups
# So that not need to use whole thing, but also get a random uniform sample from it anyways
used_positions = list(np.random.randint(0, 550000, size=20000))

source_generator_configuration = {
'seed': 1337,
'batch_size': 32,
Expand All @@ -32,7 +37,7 @@
'train_fraction': 0.6,
'validate_fraction': 0.2,
'mode': 'train',
'samples': 550000,
'samples': used_positions,
'chunked': False,
'augment': True,
}
Expand Down Expand Up @@ -67,12 +72,12 @@
'strides_lstm': 1,
'num_fc': 2,
'pooling': True,
'neurons': [32, 16, 8, 16, 32, 48],
'neurons': [32, 32, 32, 64, 32, 48],
'shape': [25, 38, 38, 1],
'start_slice': 0,
'number_slices': 25,
'activation': 'relu',
'name': 'testDisp',
'patience': 200,
}

sign_model_configuration = {
Expand Down Expand Up @@ -109,11 +114,9 @@
disp_model.test_generator = disp_test

disp_model.train()
disp_model.apply()

sign_model.train_generator = sign_train
sign_model.validate_generator = sign_validate
sign_model.test_generator = sign_test

sign_model.train()
sign_model.apply()
7 changes: 4 additions & 3 deletions factnn/data/augment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def get_random_hdf5_chunk(start, stop, size, time_slice, total_slices, gamma, pr


def get_completely_random_hdf5(start, stop, size, time_slice, total_slices, gamma, proton_input=None, labels=None,
type_training=None, augment=True, swap=True, shape=None):
augment=True, swap=True, shape=None):
'''
Gets a random part of the HDF5 database within start and stop endpoints
This is to help with shuffling data, as currently all the ones come and go in the same
Expand Down Expand Up @@ -139,9 +139,9 @@ def get_completely_random_hdf5(start, stop, size, time_slice, total_slices, gamm


def get_random_from_list(indicies, size, time_slice, total_slices, gamma, proton_input=None, labels=None,
type_training=None, augment=True, swap=True, shape=None):
augment=True, swap=True, shape=None):
'''
Gets a random part of the HDF5 database within start and stop endpoints
Gets a random part of the HDF5 database within a list of given indicies
This is to help with shuffling data, as currently all the ones come and go in the same
order
Does not guarantee that a given event will be used though, unlike before
Expand Down Expand Up @@ -174,6 +174,7 @@ def get_random_from_list(indicies, size, time_slice, total_slices, gamma, proton
return common_step(batch_images, positions, labels=labels, proton_images=proton_images, augment=augment, swap=swap, shape=shape)
else:
training_data = images_one["Image"]
positions = sorted(positions)
batch_images = training_data[positions, time_slice:time_slice + total_slices, ::]
return common_step(batch_images, positions, labels=labels, augment=augment, swap=swap, shape=shape)

Expand Down
122 changes: 78 additions & 44 deletions factnn/data/base_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ def __init__(self, config):
self.items = config['samples']
self.mode = config['mode']

# TODO Actually do something with the train and validate fractions

if 'chunked' in config:
self.chunked = config['chunked']
else:
Expand Down Expand Up @@ -71,53 +73,88 @@ def __next__(self):
:return:
'''
if not self.from_directory:
if self.chunked:
if self.mode == "train":
while True:
batch_images, batch_image_label = get_random_hdf5_chunk(0, self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label
elif self.mode == "validate":
while True:
batch_images, batch_image_label = get_random_hdf5_chunk(0, self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label

elif self.mode == "test":
while True:
batch_images, batch_image_label = get_random_hdf5_chunk(0, self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label
else:
# not chunked
if type(self.items) is int:
if self.chunked:
if self.mode == "train":
while True:
batch_images, batch_image_label = get_random_hdf5_chunk(0, self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label
elif self.mode == "validate":
while True:
batch_images, batch_image_label = get_random_hdf5_chunk(0, self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label

elif self.mode == "test":
while True:
batch_images, batch_image_label = get_random_hdf5_chunk(0, self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label
else:
# not chunked
if self.mode == "train":
while True:
batch_images, batch_image_label = get_random_from_list(self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label
elif self.mode == "validate":
while True:
batch_images, batch_image_label = get_random_from_list(self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label

elif self.mode == "test":
while True:
batch_images, batch_image_label = get_random_from_list(self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label
elif type(self.items) is list:
if self.mode == "train":
while True:
batch_images, batch_image_label = get_random_from_list(self.items, size=self.batch_size,
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
Expand All @@ -129,7 +166,6 @@ def __next__(self):
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
Expand All @@ -142,14 +178,12 @@ def __next__(self):
time_slice=self.start_slice,
total_slices=self.number_slices,
labels=self.labels,
type_training=self.type_gen,
augment=self.augment,
gamma=self.input,
proton_input=self.second_input,
shape=self.input_shape)
return batch_images, batch_image_label


def __str__(self):
return NotImplemented

Expand Down
2 changes: 1 addition & 1 deletion factnn/models/energy_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def init(self):
self.model_type = "Energy"
self.auc = 0.0
if self.name is None:
self.name = self.model_type + "_" + self.num_lstm + "LSTM_" + self.num_conv3d + "Conv3D_" + self.num_fc + \
self.name = self.model_type + "_" + str(self.num_lstm) + "LSTM_" + str(self.num_conv3d) + "Conv3D_" + str(self.num_fc) + \
"FC" + ".hdf5"

def create(self):
Expand Down
2 changes: 1 addition & 1 deletion factnn/models/source_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def init(self):
self.model_type = "Source"
self.auc = 0.0
if self.name is None:
self.name = self.model_type + "_" + self.num_lstm + "LSTM_" + self.num_conv3d + "Conv3D_" + self.num_fc + \
self.name = self.model_type + "_" + str(self.num_lstm) + "LSTM_" + str(self.num_conv3d) + "Conv3D_" + str(self.num_fc) + \
"FC" + ".hdf5"

def create(self):
Expand Down

0 comments on commit acb14e7

Please sign in to comment.