Merge pull request #5 from GlobalFishingWatch/wide

Merge latest changes in Wide to master

track_based_models/base_model.py

@@ -1,5 +1,6 @@
 from __future__ import division
 from __future__ import print_function
+import logging
 import numpy as np
 import h5py
 import os
@@ -117,8 +118,15 @@ def predict_set_times(self, data, max_deltas=0):
         for angle in [77, 167, 180, 270]:
             features, times = self.create_features_and_times(data, angle=angle,
                                         max_deltas=max_deltas)
-            predictions_for_angle = np.concatenate(self.predict(features))
-            predictions.append(predictions_for_angle)
+            try:
+                predictions_for_angle = np.concatenate(self.predict(features))
+                predictions.append(predictions_for_angle)
+            except:
+                logging.debug('prediction failed: \n' +
+                              'np.shape(features): {}\n'.format(np.shape(features)) +  
+                              'np.shape(data): {}\n'.format(np.shape(data))
+                              )
+                raise
         return times, np.mean(predictions, axis=0) > 0.5
 
     def augment_data_with_predictions(self, data):

track_based_models/peru_models.py

@@ -1,60 +1,115 @@
 from __future__ import division
 from __future__ import print_function
+import datetime
+import numpy as np
+import pandas as pd
+import os
 import keras
 from keras.models import Model as KerasModel
-from keras.layers import Dense, Dropout, Flatten, ELU, Input, Conv1D
-from keras.layers.core import Activation
+from keras.layers import Dense, Dropout, Flatten, ELU, ReLU, Input, Conv1D
+from keras.layers import BatchNormalization, MaxPooling1D, Concatenate
+from keras.layers import Cropping1D, AveragePooling1D
+from keras.layers.core import Activation, Reshape
 from keras import optimizers
 from .util import hour, minute
-from .base_model import BaseModel, hybrid_pool_layer
+from .base_model import hybrid_pool_layer_2, Normalizer, hybrid_pool_layer
 
+from .single_track_model import SingleTrackModel
+from . import util
+from .util import minute, lin_interp, cos_deg, sin_deg 
 
-class ConvNetModel5(BaseModel):
+
+from keras.engine import Layer, InputSpec
+from keras import initializers
+from keras import regularizers
+from keras import constraints
+from keras import backend as K
+
+from keras.utils.generic_utils import get_custom_objects
+
+
+
+class ConvNetModel5(SingleTrackModel):
+
     delta = 15 * minute
     window = (29 + 4*6) *  delta
     time_points = window // delta
-    base_filter_count = 32
-    fc_nodes = 512
+
+    base_filter_count = 8
+    fc_nodes = 32
+
+    data_source_lbl='fishing' 
+    data_target_lbl='is_target_encounter'
+    data_undefined_vals = (0, 3)
+    data_defined_vals = (1, 2)
+    data_true_vals = (1,)
+    data_false_vals = (2,)
+    data_far_time = 3 * 10 * minute
+
+    time_point_delta = 10000000000 # None, breaks downstream
 
     def __init__(self):
 
         self.normalizer = None
 
         depth = self.base_filter_count
 
-        input_layer = Input(shape=(self.time_points, 5))
+        input_layer = Input(shape=(self.time_points, 6))
         y = input_layer
-        y = Conv1D(depth, 3)(y)
+        y = Conv1D(depth, 4)(y)
         y = ELU()(y)
         y = keras.layers.BatchNormalization(scale=False, center=False)(y)
+        y = Dropout(0.3)(y)
         y = Conv1D(depth, 3)(y)
         y = ELU()(y)
         y = keras.layers.BatchNormalization(scale=False, center=False)(y)
         y = hybrid_pool_layer(y)
-
+        y = Dropout(0.4)(y)
+
         depth = 2 * depth 
         y = Conv1D(depth, 3)(y)
         y = ELU()(y)
         y = keras.layers.BatchNormalization(scale=False, center=False)(y)
+        y = Dropout(0.4)(y)
         y = Conv1D(depth, 3)(y)
         y = ELU()(y)
         y = keras.layers.BatchNormalization(scale=False, center=False)(y)
         y = hybrid_pool_layer(y)
-
-        y = Flatten()(y)
-        y = Dense(self.fc_nodes)(y)
+        y = Dropout(0.5)(y)
+
+        y = Conv1D(self.fc_nodes, 10, activation=None)(y)
+        # 1 - 2 - 6 - 12 - 17 + 4 * (n - 1)
+
+        # y = Flatten()(y)
+        # y = Dense(self.fc_nodes)(y)
         y = ELU()(y)
         y = keras.layers.BatchNormalization(scale=False, center=False)(y)
 
-        y = Dense(self.fc_nodes)(y)
+        # y = Dense(self.fc_nodes)(y)
+        y = Conv1D(self.fc_nodes, 1, activation=None)(y)
         y = ELU()(y)
         y = Dropout(0.5)(y)
 
-        y = Dense(1)(y)
+        y = Conv1D(1, 1, activation=None)(y)
         y = Activation('sigmoid')(y)
         output_layer = y
         model = KerasModel(inputs=input_layer, outputs=output_layer)
         opt = optimizers.Nadam(lr=0.0005, schedule_decay=0.01)
         #opt = optimizers.Adam(lr=0.01, decay=0.5)
-        model.compile(optimizer=opt, loss='binary_crossentropy', metrics=["accuracy"])
-        self.model = model  
+        model.compile(optimizer=opt, loss='binary_crossentropy', metrics=["accuracy"],
+            sample_weight_mode="temporal")
+        self.model = model 
+
+    def fit(self, x, labels, epochs=1, batch_size=32, sample_weight=None, 
+            validation_split=0, validation_data=0, verbose=1, callbacks=[]):
+        self.normalizer = Normalizer().fit(x)
+        x1 = self.preprocess(x)
+        l1 = np.asarray(labels).reshape(len(labels), -1, 1)
+        if validation_data not in (None, 0):
+            a, b, c = validation_data
+            validation_data = self.preprocess(a), b, c
+        return self.model.fit(x1, l1, epochs=epochs, batch_size=batch_size, 
+                        sample_weight=sample_weight,
+                      validation_split=validation_split, 
+                      validation_data=validation_data,
+                      verbose=verbose, callbacks=callbacks)

track_based_models/single_track_model.py

@@ -39,6 +39,8 @@ def create_features_and_times(self, data, angle=77, max_deltas=0):
     def build_features(cls, obj, skip_label=False, keep_frac=1.0):
         delta = cls.delta
         n_pts = len(obj['lat'])
+        if n_pts == 0:
+            return [], [], [], [], []
         assert 0 < keep_frac <= 1, 'keep frac must be between 0 and 1'
         if keep_frac == 1:
             mask = None
@@ -139,11 +141,12 @@ def load_data(cls, path, delta, skip_label=False, keep_fracs=[1], features=None,
                          vessel_label=None):
         obj_tv = util.load_json_data(path, vessel_label=vessel_label)  
         obj = util.convert_from_legacy_format(obj_tv)
-        obj[cls.data_source_lbl] = obj_tv[cls.data_source_lbl]
+        mask = ~np.isnan(np.array(obj_tv['sogs'])) & ~np.isnan(np.array(obj_tv['courses']))
+        obj[cls.data_source_lbl] = np.asarray(obj_tv[cls.data_source_lbl])[mask]
         # if features is None:
         if features is not None:
             # Filter features down to just the ssvid / time span we want
-            ssvid = os.path.basename(path).split('_')[0]
+            ssvid = obj_tv['mmsi']
             mask = (features.ssvid == ssvid)
             features = features[mask]
             features = features.sort_values(by='timestamp')
@@ -168,7 +171,8 @@ def load_data(cls, path, delta, skip_label=False, keep_fracs=[1], features=None,
                 t, x, y, label, is_defined = cls.build_features(obj, 
                                                 skip_label=skip_label, keep_frac=kf)
             except:
-                print('skipping', path, kf)
+                raise
+                print('skipping', path, kf, 'due to unknown error')
                 continue
             t = np.asarray(t)
             yield (t, x, y, label, is_defined)
@@ -229,7 +233,7 @@ def generate_data(cls, paths, min_samples, seed=888,
                                     features=precomp_features, 
                                     vessel_label=vessel_label):
                 if data is None:
-                    print('skipping', p)
+                    print('skipping', p, 'because data is None')
                     continue
                 (t, x, y, label, dfnd) = data
 

track_based_models/util.py

@@ -39,9 +39,10 @@ def load_json_data(path, vessel_label):
         obj = obj[vessel_label]
     obj['raw_timestamps'] = obj['timestamps']
     obj['timestamps'] = obj['timestamp'] = [dateutil.parser.parse(x) for x in obj['timestamps']]
-    mask = np.ones_like(obj['timestamp'])
+    mask = np.ones_like(obj['timestamp'], dtype=bool)
     for field in ['sogs', 'courses']:
-        mask &= [(x is not None) for x in obj[field]]
+#	print(mask.dtype,  np.array([(x is not None) for x in obj[field]]).dtype)
+        mask &= np.array([(x is not None) for x in obj[field]], dtype=bool)
     for field in ['timestamp', 'lats', 'lons', 'sogs', 'courses']:
         obj[field] = [x for (i, x) in enumerate(obj[field]) if mask[i]]
     return obj