Start reorganization

Moving things around and starting update to split functionality among more files for easier maintanance and better modularity. Also make the import paths make more sense.

factnn/__init__.py

@@ -1,9 +1,6 @@
 from factnn.models.separation_models import SeparationModel
 from factnn.models.source_models import DispModel, SignModel
 from factnn.models.energy_models import EnergyModel
-from factnn.preprocess.simulation_preprocessors import GammaDiffusePreprocessor, GammaPreprocessor, ProtonPreprocessor
-from factnn.preprocess.observation_preprocessors import ObservationPreprocessor
-from factnn.data.separation_generators import SeparationGenerator
-from factnn.data.energy_generators import EnergyGenerator
-from factnn.data.source_generators import DispGenerator, SignGenerator
+from factnn.data.preprocess.simulation_preprocessors import GammaDiffusePreprocessor, GammaPreprocessor, ProtonPreprocessor
+from factnn.generator.generator.source_generators import DispGenerator, SignGenerator
 from factnn.utils import plotting

factnn/preprocess/observation_preprocessors.py → factnn/data/preprocess/observation_preprocessors.py

@@ -4,7 +4,7 @@
 from fact.io import read_h5py
 import datetime
 
-from factnn.preprocess.base_preprocessor import BasePreprocessor
+from factnn.data.preprocess.base_preprocessor import BasePreprocessor
 
 
 class ObservationPreprocessor(BasePreprocessor):

factnn/preprocess/simulation_preprocessors.py → factnn/data/preprocess/simulation_preprocessors.py

@@ -2,7 +2,7 @@
 import h5py
 import photon_stream as ps
 from fact.io import read_h5py
-from factnn.preprocess.base_preprocessor import BasePreprocessor
+from factnn.data.preprocess.base_preprocessor import BasePreprocessor
 from sklearn.utils import shuffle
 
 

factnn/data/base_generator.py → factnn/generator/generator/base_generator.py

@@ -1,4 +1,4 @@
-from factnn.data.augment import get_random_from_list, \
+from factnn.utils.augment import get_random_from_list, \
     get_chunk_from_list, get_random_from_paths
 import numpy as np
 

factnn/data/energy_generators.py → factnn/generator/generator/energy_generators.py

@@ -1,4 +1,4 @@
-from factnn.data.base_generator import BaseGenerator
+from factnn.generator.generator.base_generator import BaseGenerator
 import h5py
 
 class EnergyGenerator(BaseGenerator):

factnn/data/separation_generators.py → factnn/generator/generator/separation_generators.py

@@ -1,4 +1,4 @@
-from factnn.data.base_generator import BaseGenerator
+from factnn.generator.generator.base_generator import BaseGenerator
 import h5py
 
 class SeparationGenerator(BaseGenerator):

factnn/data/source_generators.py → factnn/generator/generator/source_generators.py

@@ -1,4 +1,4 @@
-from factnn.data.base_generator import BaseGenerator
+from factnn.generator.generator.base_generator import BaseGenerator
 import h5py
 import numpy as np
 

factnn/data/sequence_generator.py → factnn/generator/keras/sequence_generator.py

@@ -48,6 +48,33 @@ def __init__(self, paths, num_elements, batch_size, preprocessor=None, proton_pr
         Otherwise, the validation data could be set to use the first few elements from each file instead, the batch size,
         but this makes the validation set much smaller than otherwise usually used
 
+
+        Final setup then would be:
+
+        BaseGenerator for HDF5 files, streaming in files with infinite loops if no need for multiprocesssing
+        SequenceGenerator for streaming in files, with each file used once per iteration, regardless of how much of it is used
+        EventGenerator for streaming in files where there is one event per file, still a form of Sequence Generator,
+        but designed for a single event per file setup
+
+        Other things needed:
+        Preprocessor on_batch_preprocess for SequenceGenerator
+        Way to generate the one event per file setup
+        New class of DataFormatter?
+
+        Dataset class that holds the parts for creating the HDF5 dataset, one file per event conversion, and
+
+        New Design:
+            models -> holds most useful models
+            docs -> docs
+            resources -> resources
+            tests -> tests
+            utils -> augment, plotting parts
+            data -> data.preprocess holds preprocessors
+            data -> data.dataset holds creating HDF5 converthing to different storage formats
+            generator -> generator.keras holds Sequence-based generators
+            generator -> generator.generator holds BaseGenerator-based generators
+
+
         :param paths:
         :param num_elements:
         :param batch_size: Maximum size of batch, might be smaller depending on how many events are left in each file

factnn/tests/test_generators.py

@@ -1,7 +1,7 @@
 import unittest
-from factnn.data.energy_generators import EnergyGenerator
-from factnn.data.separation_generators import SeparationGenerator
-from factnn.data.source_generators import DispGenerator, SignGenerator
+from factnn.generator.generator.energy_generators import EnergyGenerator
+from factnn.generator.generator.separation_generators import SeparationGenerator
+from factnn.generator.generator.source_generators import DispGenerator, SignGenerator
 
 
 class TestEnergyGenerator(unittest.TestCase):

factnn/tests/test_preprocessing.py

@@ -1,7 +1,6 @@
 import unittest
 
-from factnn.preprocess.observation_preprocessors import ObservationPreprocessor
-from factnn.preprocess.simulation_preprocessors import ProtonPreprocessor, GammaPreprocessor
+from factnn.data.preprocess.simulation_preprocessors import GammaPreprocessor
 
 
 class TestProtonPreprocessor(unittest.TestCase):