Start on streaming from files

Going back, now the generators will use the preprocessors to preprocess the image, the generators will then just call the single preprocess as needed for elements in the array, with a single preprocessor for each the training, test, and validation datasets
jacobbieker · Oct 30, 2018 · 20b378a · 20b378a
1 parent 8963f22
commit 20b378a
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 5 deletions.
diff --git a/examples/energy.py b/examples/energy.py
@@ -83,7 +83,7 @@
 energy_model.validate_generator = energy_validate
 energy_model.test_generator = energy_test
 
-energy_model.train()
+energy_model.train(train_generator=energy_train, validate_generator=energy_validate)
 energy_model.apply()
 
 
diff --git a/factnn/preprocess/observation_preprocessors.py b/factnn/preprocess/observation_preprocessors.py
@@ -105,7 +105,8 @@ def single_processor(self):
                         data.append([np.fliplr(np.rot90(input_matrix, 3)), energy, zd_deg, az_deg, source_pos_x,
                                      source_pos_y, sky_source_zd, sky_source_az, zd_deg1, az_deg1,
                                      event_num, night, run, cog_x, cog_y])
-                    yield data
+                    # need to do the format thing here
+                    yield self.format(data)
 
             except Exception as e:
                 print(str(e))

diff --git a/factnn/preprocess/simulation_preprocessors.py b/factnn/preprocess/simulation_preprocessors.py
@@ -72,7 +72,7 @@ def single_processor(self):
                                 if self.end > value > self.start:
                                     input_matrix[coords[0]][coords[1]][value - self.start] += element[1] * 1
                     data.append([np.fliplr(np.rot90(input_matrix, 3)), energy, zd_deg, az_deg, act_phi, act_theta])
-                    yield data
+                    yield self.format(data)
 
             except Exception as e:
                 print(str(e))
@@ -208,7 +208,7 @@ def single_processor(self):
                                 if self.end > value > self.start:
                                     input_matrix[coords[0]][coords[1]][value - self.start] += element[1] * 1
                     data.append([np.fliplr(np.rot90(input_matrix, 3)), energy, zd_deg, az_deg, act_phi, act_theta])
-                    yield data
+                    yield self.format(data)
 
             except Exception as e:
                 print(str(e))
@@ -374,7 +374,7 @@ def single_processor(self):
                         data.append([np.fliplr(np.rot90(input_matrix, 3)), act_sky_source_zero, act_sky_source_one,
                                      cog_x, cog_y, zd_deg, az_deg, sky_source_zd, sky_source_az, delta,
                                      energy, zd_deg1, az_deg1])
-                    yield data
+                    yield self.format(data)
 
             except Exception as e:
                 print(str(e))