how to use fit_generator with multiple image inputs

[aendrs]

Hello, I'm trying to use a model with paired input images through (in their own similar directory trees), augmented through ImageDataGenerator using also flow_from_directory (so the method infers the labels by the folder structure). I'm getting an error because keras can't handle it in this way.

How can I combine the generators (using flow_from_directory) to be accepted by fit_generator?

Here is a sample code

Model definition ***

#two classic CNN blocks are defined before these lines and then cocatenated
#create model
model = Model([input_img_1,input_img_2], out, name='colliculus_proto')

#compile model
model.compile(optimizer=Adam(lr=1e-3), 
              loss='categorical_crossentropy', 
              metrics=['accuracy'])

# image data generators for image inputs
def input_generator(train_dir,batchsize,img_height,img_width):
    train_generator = ImageDataGenerator(rescale = 1./255, 
                                       shear_range = 0.2, 
                                       zoom_range = 0.2,
                                       rotation_range=5.,
                                       horizontal_flip = True)
    training_set = train_generator.flow_from_directory(train_dir,
                                                 target_size = (img_height,img_width),
                                                 class_mode = 'categorical',
                                                 batch_size = batchsize,
                                                 shuffle=False)
    return training_set
                                             

def test_generator(test_dir,batchsize,img_height,img_width):
    test_gen = ImageDataGenerator(rescale = 1./255)
    test_set = test_gen.flow_from_directory(test_dir,
                                            target_size = (img_height,img_width),
                                            class_mode = 'categorical',
                                            batch_size = batchsize,
                                            shuffle=False)
    return test_set


#fit model
input1=input_generator(train_dir_1,batchsize,img_height,img_width)
input2=input_generator(train_dir_2,batchsize,img_height,img_width)
test1=test_generator(test_dir_1,batchsize,img_height,img_width)
test2=test_generator(test_dir_2,batchsize,img_height,img_width)

model.fit_generator([input1,input2],
                        steps_per_epoch=trainsetsize/batchsize,
                        epochs = epochs,
                        validation_data = [test1,test2],
                        validation_steps = testsetsize/batchsize,
                        use_multiprocessing=True,
                        shuffle=False)

The error I get is the following:

TypeError: Error when checking model input: data should be a Numpy array, or list/dict of Numpy arrays. Found: <keras.preprocessing.image.DirectoryIterator object at 0x7f824c5080f0>...

[wangqianwen0418]

https://github.com/fchollet/keras/issues/3386
this might help.
I think you need a generator that yields something of the form ([x1, x2], y).

[aendrs]

Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful

input_imgen = ImageDataGenerator(rescale = 1./255, 
                                   shear_range = 0.2, 
                                   zoom_range = 0.2,
                                   rotation_range=5.,
                                   horizontal_flip = True)

test_imgen = ImageDataGenerator(rescale = 1./255)



def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
    genX1 = generator.flow_from_directory(dir1,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    
    genX2 = generator.flow_from_directory(dir2,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    while True:
            X1i = genX1.next()
            X2i = genX2.next()
            yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
            
            
inputgenerator=generate_generator_multiple(generator=input_imgen,
                                           dir1=train_dir_1,
                                           dir2=train_dir_2,
                                           batch_size=batch_size,
                                           img_height=img_height,
                                           img_width=img_height)       
     
testgenerator=generate_generator_multiple(test_imgen,
                                          dir1=train_dir_1,
                                          dir2=train_dir_2,
                                          batch_size=batch_size,
                                          img_height=img_height,
                                          img_width=img_height)              
          
 history=model.fit_generator(inputgenerator,
                        steps_per_epoch=trainsetsize/batch_size,
                        epochs = epochs,
                        validation_data = testgenerator,
                        validation_steps = testsetsize/batch_size,
                        use_multiprocessing=True,
                        shuffle=False)

[ghost]

That was great. How would you find the class_indices for the inputgenerator and testgenerator? I'm not able to use this syntax: inputgenerator.class_indices() here since now, the generator works with multiple inputs.

[laukun]

@aendrs I would like to ask how to give the two inputs same pre-processing everytime? As in the function of input_generator, input_imgen is called twice, for each time randomly selected preprocessing steps are applied on the input.

[MjdMahasneh]

@aendrs @raaju-shiv @wangqianwen0418 @laukun @bmabey Could anyone of you guys kindly help me solve this problem #10499. I tried implementing the same generator as in this post but i dont seem to figure out where is my mistake. any help is very much appreciated.

[sdw95927]

@laukun I would suggest write your own data generator like in this example:

class DataGenerator(keras.utils.Sequence):
    """Generates data for Keras."""
    def __init__(self, img_files, clinical_info, labels, ave=None, std=None, batch_size=32, dim=(300, 300), n_channels=3,
                 n_classes=2, shuffle=True):
        """Initialization.
        
        Args:
            img_files: A list of path to image files.
            clinical_info: A dictionary of corresponding clinical variables.
            labels: A dictionary of corresponding labels.
        """
        self.img_files = img_files
        self.clinical_info = clinical_info
        self.labels = labels
        self.batch_size = batch_size
        self.dim = dim
        if ave is None:
            self.ave = np.zeros(n_channels)
        else:
            self.ave = ave
        if std is None:
            self.std = np.zeros(n_channels) + 1
        else:
            self.std = std
        
        self.n_channels = n_channels
        self.n_classes = n_classes
        self.shuffle = shuffle
        self.on_epoch_end()

    def __len__(self):
        """Denotes the number of batches per epoch."""
        return int(np.floor(len(self.img_files) / self.batch_size))

    def __getitem__(self, index):
        """Generate one batch of data."""
        # Generate indexes of the batch
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]

        # Find list of IDs
        img_files_temp = [self.img_files[k] for k in indexes]

        # Generate data
        X, y = self.__data_generation(img_files_temp)

        return X, y

    def on_epoch_end(self):
        """Updates indexes after each epoch."""
        self.indexes = np.arange(len(self.img_files))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self, img_files_temp):
        """Generates data containing batch_size samples."""
        # X : (n_samples, *dim, n_channels)
        # X = [np.empty((self.batch_size, self.dim[0], self.dim[1], self.n_channels))]
        X_img = []
        X_clinical = []
        y = np.empty((self.batch_size), dtype=int)

        # Generate data
        for i, img_file in enumerate(img_files_temp):
            # Read image
            img = skimage.io.imread(img_file)
            
            # Resize
            img = skimage.transform.resize(img, output_shape=self.dim, mode='constant', preserve_range=True)
            
            # Normalization
            for ch in range(self.n_channels):
                img[:, :, ch] = (img[:, :, ch] - self.ave[ch])/self.std[ch]
            
            if self.shuffle:
                # Some image augmentation codes
                ###### You can put your preprocessing codes here. #####

            X_img.append(img)
            X_clinical.append(self.clinical_info[img_file])
            y[i] = self.labels[img_file]
        X = [np.array(X_img), np.array(X_clinical)]
        return X, keras.utils.to_categorical(y, num_classes=self.n_classes)

And to call the generator:

train_datagen = DataGenerator(img_files=img_files, clinical_info=clinical_info, labels=labels, ave=ave, std=std, batch_size=batch_size)
val_datagen = DataGenerator(img_files=img_files, clinical_info=clinical_info, labels=labels, ave=ave, std=std, batch_size=batch_size)
hist = model.fit_generator(train_datagen, 
                           steps_per_epoch=len(img_files) / batch_size, 
                           epochs=num_epochs, 
                           verbose=1, 
                           validation_data=val_datagen, 
                           validation_steps=1)

The different thing I'm doing here is using two different kinds of input, an image and a numpy array. I saved image paths as a list, and created two dictionaries whose key words are the image paths. When you want to do the preprocessing, you can easily apply them right after reading the image using the image paths.

But this way, the speed will be definitely slower than loading data directly from .npy

[scstu]

I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`

Arch

def fusionNet(nb_classes, inputs=(3, 224, 224)):
    input_img = Input(shape=inputs, name='RGB_input')
    conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", 
    ## ....
    ## ....
    ## ....
    firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout)
    ##
    print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13)



    input_img2 = Input(shape=inputs, name='depth_input')
    conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same",             kernel_initializer="glorot_uniform")(input_img2)
   #....
   #....
   #...
## BN
    SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2)
    print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13)
    #     

    merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1)   
    ### got here (None, 1024, 13, 13) 
    conv10 = Conv2D(
    nb_classes, (1, 1),init='glorot_uniform',
    border_mode='valid', name='conv10')(merge2CNN)
   avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10)

    flatten = Flatten(name='flatten')(avgpool10)
   softmax = Activation("softmax", name='softmax')(flatten)

    return Model(inputs=[input_img,input_img2], output=softmax)

`

Training

sn = fusionNet(nb_classes=24)
sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True)

sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy')

train_datagen = ImageDataGenerator(
    rescale=1./255,
    zoom_range=0.01,
    rotation_range=2.,
    featurewise_center=True,
    featurewise_std_normalization=True,
)
#train_datagen = ImageDataGenerator(rescale=1./255)

test_datagen = ImageDataGenerator(rescale=1./255)


def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
                                  target_size = (img_height,img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size,
                                  shuffle=False)

genX2 = generator.flow_from_directory(dir2,
                                  target_size = (img_height, img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size)
while True:
    X1i = genX1.next()
    X2i = genX2.next()
    yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
    
    
train_generator=generate_generator_multiple(generator=train_datagen,
                                   dir1=train_data_dir,
                                   dir2=train_data_dir_colored,
                                   batch_size=32,
                                   img_height=size,
                                   img_width=size)       

validation_generator=generate_generator_multiple(test_datagen,
                                  dir1=validation_data_dir,
                                  dir2=validation_data_dir_colored,
                                  batch_size=batch_size,
                                  img_height=size,
                                  img_width=size)              



tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0,         batch_size=32, write_graph=True)
sn.fit_generator(
        train_generator,
    steps_per_epoch=nb_train_samples//32,
    nb_epoch=nb_epoch,
    validation_data=validation_generator,
    validation_steps = nb_validation_samples//32, 
    callbacks=[tensorboardcb])

[wkecn]

@scstu I would like to merge the input data first, which might be a better idea.

[MjdMahasneh]

@scstu that can be done in different ways. but keep in mind the number of learned parameters differ depending on the way you merge the input images. so if youre merging in a multispectral-like shape (for 2xRGB images you can merge in two ways as far as i know. so it would be something like: HxWx6 presuming that your concatenating the images, or it could be HxWx3 if youre finding a transformation of the 2 RGB images such as multiplication, add, sub etc..) in either case be careful with the input size for your model and remember that the number of the learned parameters will differ. to know whats better you can only try and compare the results. it also will affect the extracted feature by the convolution differently. i worked on a similar problem and i used multiple branch CNN and 2 input images where each images passes a branch from the CNN and then the features extracted by the two CNN branches are merged in a merging layer before passing it to the top layers.

[gledsonmelotti]

@aendrs thank you very much on your algorithm. He is fantastic.

[aamirk5]

@aendrs it was really good work to generate multi-streams input. thanks

[aamirk5]

@aendrs could you please share your portion of code for concatenation and inputs manipulation

[gledsonmelotti]

I have a doubt. I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).

[gledsonmelotti]

@scstu that can be done in different ways. but keep in mind the number of learned parameters differ depending on the way you merge the input images. so if youre merging in a multispectral-like shape (for 2xRGB images you can merge in two ways as far as i know. so it would be something like: HxWx6 presuming that your concatenating the images, or it could be HxWx3 if youre finding a transformation of the 2 RGB images such as multiplication, add, sub etc..) in either case be careful with the input size for your model and remember that the number of the learned parameters will differ. to know whats better you can only try and compare the results. it also will affect the extracted feature by the convolution differently. i worked on a similar problem and i used multiple branch CNN and 2 input images where each images passes a branch from the CNN and then the features extracted by the two CNN branches are merged in a merging layer before passing it to the top layers.

@MjdMahasneh I believe your comment is interesting. Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).

[gledsonmelotti]

I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`

Arch

def fusionNet(nb_classes, inputs=(3, 224, 224)):
    input_img = Input(shape=inputs, name='RGB_input')
    conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", 
    ## ....
    ## ....
    ## ....
    firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout)
    ##
    print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13)



    input_img2 = Input(shape=inputs, name='depth_input')
    conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same",             kernel_initializer="glorot_uniform")(input_img2)
   #....
   #....
   #...
## BN
    SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2)
    print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13)
    #     

    merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1)   
    ### got here (None, 1024, 13, 13) 
    conv10 = Conv2D(
    nb_classes, (1, 1),init='glorot_uniform',
    border_mode='valid', name='conv10')(merge2CNN)
   avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10)

    flatten = Flatten(name='flatten')(avgpool10)
   softmax = Activation("softmax", name='softmax')(flatten)

    return Model(inputs=[input_img,input_img2], output=softmax)

`

Training

sn = fusionNet(nb_classes=24)
sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True)

sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy')

train_datagen = ImageDataGenerator(
    rescale=1./255,
    zoom_range=0.01,
    rotation_range=2.,
    featurewise_center=True,
    featurewise_std_normalization=True,
)
#train_datagen = ImageDataGenerator(rescale=1./255)

test_datagen = ImageDataGenerator(rescale=1./255)


def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
                                  target_size = (img_height,img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size,
                                  shuffle=False)

genX2 = generator.flow_from_directory(dir2,
                                  target_size = (img_height, img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size)
while True:
    X1i = genX1.next()
    X2i = genX2.next()
    yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
    
    
train_generator=generate_generator_multiple(generator=train_datagen,
                                   dir1=train_data_dir,
                                   dir2=train_data_dir_colored,
                                   batch_size=32,
                                   img_height=size,
                                   img_width=size)       

validation_generator=generate_generator_multiple(test_datagen,
                                  dir1=validation_data_dir,
                                  dir2=validation_data_dir_colored,
                                  batch_size=batch_size,
                                  img_height=size,
                                  img_width=size)              



tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0,         batch_size=32, write_graph=True)
sn.fit_generator(
        train_generator,
    steps_per_epoch=nb_train_samples//32,
    nb_epoch=nb_epoch,
    validation_data=validation_generator,
    validation_steps = nb_validation_samples//32, 
    callbacks=[tensorboardcb])

Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).

[AndreAbade]

I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`

Arch

def fusionNet(nb_classes, inputs=(3, 224, 224)):
    input_img = Input(shape=inputs, name='RGB_input')
    conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", 
    ## ....
    ## ....
    ## ....
    firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout)
    ##
    print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13)



    input_img2 = Input(shape=inputs, name='depth_input')
    conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same",             kernel_initializer="glorot_uniform")(input_img2)
   #....
   #....
   #...
## BN
    SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2)
    print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13)
    #     

    merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1)   
    ### got here (None, 1024, 13, 13) 
    conv10 = Conv2D(
    nb_classes, (1, 1),init='glorot_uniform',
    border_mode='valid', name='conv10')(merge2CNN)
   avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10)

    flatten = Flatten(name='flatten')(avgpool10)
   softmax = Activation("softmax", name='softmax')(flatten)

    return Model(inputs=[input_img,input_img2], output=softmax)

`

Training

sn = fusionNet(nb_classes=24)
sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True)

sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy')

train_datagen = ImageDataGenerator(
    rescale=1./255,
    zoom_range=0.01,
    rotation_range=2.,
    featurewise_center=True,
    featurewise_std_normalization=True,
)
#train_datagen = ImageDataGenerator(rescale=1./255)

test_datagen = ImageDataGenerator(rescale=1./255)


def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
                                  target_size = (img_height,img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size,
                                  shuffle=False)

genX2 = generator.flow_from_directory(dir2,
                                  target_size = (img_height, img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size)
while True:
    X1i = genX1.next()
    X2i = genX2.next()
    yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
    
    
train_generator=generate_generator_multiple(generator=train_datagen,
                                   dir1=train_data_dir,
                                   dir2=train_data_dir_colored,
                                   batch_size=32,
                                   img_height=size,
                                   img_width=size)       

validation_generator=generate_generator_multiple(test_datagen,
                                  dir1=validation_data_dir,
                                  dir2=validation_data_dir_colored,
                                  batch_size=batch_size,
                                  img_height=size,
                                  img_width=size)              



tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0,         batch_size=32, write_graph=True)
sn.fit_generator(
        train_generator,
    steps_per_epoch=nb_train_samples//32,
    nb_epoch=nb_epoch,
    validation_data=validation_generator,
    validation_steps = nb_validation_samples//32, 
    callbacks=[tensorboardcb])

Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).

Hello,

Did you find an answer / solution to your question? I have the same problem / doubt.

[sdw95927]

I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`

Arch

def fusionNet(nb_classes, inputs=(3, 224, 224)):
    input_img = Input(shape=inputs, name='RGB_input')
    conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", 
    ## ....
    ## ....
    ## ....
    firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout)
    ##
    print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13)



    input_img2 = Input(shape=inputs, name='depth_input')
    conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same",             kernel_initializer="glorot_uniform")(input_img2)
   #....
   #....
   #...
## BN
    SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2)
    print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13)
    #     

    merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1)   
    ### got here (None, 1024, 13, 13) 
    conv10 = Conv2D(
    nb_classes, (1, 1),init='glorot_uniform',
    border_mode='valid', name='conv10')(merge2CNN)
   avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10)

    flatten = Flatten(name='flatten')(avgpool10)
   softmax = Activation("softmax", name='softmax')(flatten)

    return Model(inputs=[input_img,input_img2], output=softmax)

`

Training

sn = fusionNet(nb_classes=24)
sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True)

sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy')

train_datagen = ImageDataGenerator(
    rescale=1./255,
    zoom_range=0.01,
    rotation_range=2.,
    featurewise_center=True,
    featurewise_std_normalization=True,
)
#train_datagen = ImageDataGenerator(rescale=1./255)

test_datagen = ImageDataGenerator(rescale=1./255)


def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
                                  target_size = (img_height,img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size,
                                  shuffle=False)

genX2 = generator.flow_from_directory(dir2,
                                  target_size = (img_height, img_width),
                                  class_mode = 'categorical',
                                  batch_size = batch_size)
while True:
    X1i = genX1.next()
    X2i = genX2.next()
    yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
    
    
train_generator=generate_generator_multiple(generator=train_datagen,
                                   dir1=train_data_dir,
                                   dir2=train_data_dir_colored,
                                   batch_size=32,
                                   img_height=size,
                                   img_width=size)       

validation_generator=generate_generator_multiple(test_datagen,
                                  dir1=validation_data_dir,
                                  dir2=validation_data_dir_colored,
                                  batch_size=batch_size,
                                  img_height=size,
                                  img_width=size)              



tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0,         batch_size=32, write_graph=True)
sn.fit_generator(
        train_generator,
    steps_per_epoch=nb_train_samples//32,
    nb_epoch=nb_epoch,
    validation_data=validation_generator,
    validation_steps = nb_validation_samples//32, 
    callbacks=[tensorboardcb])

Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).

Hello,

Did you find an answer / solution to your question? I have the same problem / doubt.

I think the training size is the later one, sum of all training samples. The question is equal with:

A model has three sets of parameters A, B and C. We use X1 data to train A & C with B fixed, and use X2 to train B & C with A fixed.

I think now the answer is clear.

[aezco]

I am trying similar things, but I am getting stuck with the input of the model.

`
def generate_generator_multiple(generator,dir1, dir2,dir3, batch_size, img_size):

      genX1 = generator.flow_from_directory(dir1,
                                      target_size = (img_size,img_size),
                                      class_mode = 'categorical',
                                      batch_size = batch_size,
                                      shuffle=False, 
                                      seed=7,subset='training')

      genX2 = generator.flow_from_directory(dir2,
                                      target_size = (img_size,img_size),
                                      class_mode = 'categorical',
                                      batch_size = batch_size,
                                      shuffle=False, 
                                      seed=7,subset='training')

      genX3 = generator.flow_from_directory(dir3,
                                  target_size = (img_size,img_size),
                                  class_mode = 'categorical',
                                  batch_size = batch_size,
                                  shuffle=False, 
                                  seed=7,subset='training')
       while True:
              X1i = genX1.next()
              X2i = genX2.next()
              X3i = genX3.next()
       yield [X1i[0], X2i[0],X3i[0]], X3i[1]  #Yield both images and their mutual label

train_datagen = ImageDataGenerator(validation_split=VAL_SPLIT,rescale=1./255)
inputgenerator=generate_generator_multiple(generator=train_datagen,
                                       dir1='mergedata',
                                       dir2='mergedata',
                                       dir3='mergedata',
                                       batch_size=BATCH_SIZE,
                                       img_size=IMG_SIZE)   

modelA = load_model('/content/8-base_model_1.h5')
modelA.load_weights('/content/8-weights.002-0.991-0.406.hdf5') 

modelB = load_model('/content/30-base_model_1.h5')
modelB.load_weights('/content/30-weights.010-0.499-0.374.hdf5')

modelC = load_model('/content/80-base_model_1.h5')
modelC.load_weights('/content/80-weights.002-1.064-0.305.hdf5')
for layer in modelA.layers:
    layer.trainable = False
    layer.name = layer.name + str("_1")
  
for layer in modelB.layers:
    layer.trainable = False
    layer.name = layer.name + str("_2")
    
for layer in modelC.layers:
    layer.trainable = False
    layer.name = layer.name + str("_3")

mod_1 = concatenate([modelA.layers[-1].output,  modelB.layers[-1].output,modelC.layers[-1].output])     
mod_1 = Dropout(0.2)(mod_1)
mod_1 = Dense(15, activation='relu')(mod_1)
predictions = Dense(9, activation='softmax', name='pred_age')(mod_1)

top_model = Model(inputs=[modelA.input, modelB.input, modelC.input], outputs=predictions)

opt = get_optimizer('adam', lr_rate)
top_model.compile(loss=['categorical_crossentropy'],
          optimizer=opt,
          metrics=['accuracy','mae'])

model.fit_generator(inputgenerator,
                           epochs=EPOCHS,
                           steps_per_epoch = STEPS_PER_EPOCH,
                           validation_data=validgenerator,
                           validation_steps = VALIDATION_STEPS,
                           verbose=1,
                           callbacks=callbacks)`

I first trained 3 different models:

• model 1 = 3 labels (8,9,10)
• model 2 = 2 labels (30,31)
• model 3 = 4 labels (80,81,82,83)

I would like to combine these 3 models into a final model with 1 output containing 9 labels (8,9,10,30,31,80,81,82,83). The final input needs to get 1 input image instead of 3 images. But I still get stuck. I am building a kind of hierarchy here to improve the accuracy.

[enviz]

Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful

input_imgen = ImageDataGenerator(rescale = 1./255, 
                                   shear_range = 0.2, 
                                   zoom_range = 0.2,
                                   rotation_range=5.,
                                   horizontal_flip = True)

test_imgen = ImageDataGenerator(rescale = 1./255)



def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
    genX1 = generator.flow_from_directory(dir1,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    
    genX2 = generator.flow_from_directory(dir2,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    while True:
            X1i = genX1.next()
            X2i = genX2.next()
            yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
            
            
inputgenerator=generate_generator_multiple(generator=input_imgen,
                                           dir1=train_dir_1,
                                           dir2=train_dir_2,
                                           batch_size=batch_size,
                                           img_height=img_height,
                                           img_width=img_height)       
     
testgenerator=generate_generator_multiple(test_imgen,
                                          dir1=train_dir_1,
                                          dir2=train_dir_2,
                                          batch_size=batch_size,
                                          img_height=img_height,
                                          img_width=img_height)              
          
 history=model.fit_generator(inputgenerator,
                        steps_per_epoch=trainsetsize/batch_size,
                        epochs = epochs,
                        validation_data = testgenerator,
                        validation_steps = testsetsize/batch_size,
                        use_multiprocessing=True,
                        shuffle=False)

How is that that the test generator and train generator(input generator) have the same directories??
Shouldn't it be different??
(dir1 and dir2 should be different)
Please correct me if I'm wrong.

[zetaSaahil]

def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7)

genX2 = generator.flow_from_directory(dir2,
                                      target_size = (img_height,img_width),
                                      class_mode = 'categorical',
                                      batch_size = batch_size,
                                      shuffle=False, 
                                      seed=7)
while True:
        X1i = genX1.next()
        X2i = genX2.next()
        yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label

In this method shown,
I have a situation where I am returning two ground truth masks, and one image using a custom data_generator as shown above.
However, how do I incorporate it into a custom loss function.

def custom_loss(y_true1,ytrue2, y_pred):
    #compute loss here
    return loss

[apres91]

Hi, I am using the train generator for two inputs as @aendrs wrote.
The problem is that the two generators have two different orders, and this is not intended because we have mutual labels. Is there a way to use order of the first generator in the second as well? Thank you!

[apres91]

To reply to my comment, I made my own class of generator that takes as input 2 images and allows multiple labels:

import numpy as np
import keras
import imageio

class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def init(self, list_IDs1, list_IDs2, labels, batch_size=32, dim=(32,32,32), n_channels=1,
n_classes=10, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.labels = labels
self.list_IDs1 = list_IDs1
self.list_IDs2 = list_IDs2
self.n_channels = n_channels
self.dim=dim
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
def len(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs1) / self.batch_size))
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs1))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def getitem(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
# Find list of IDs
list_IDs_temp1 = [self.list_IDs1[k] for k in indexes]
list_IDs_temp2 = [self.list_IDs2[k] for k in indexes]
# Generate data
[X1,X2], y = self.__data_generation(list_IDs_temp1,list_IDs_temp2,indexes)
return [X1,X2], y
def __data_generation(self, list_IDs_temp1,list_IDs_temp2,indexes):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X1 = np.empty((self.batch_size, *self.dim, self.n_channels))
X2 = np.empty((self.batch_size, *self.dim, self.n_channels))
y = [np.empty(self.batch_size,dtype=np.float32) for _ in range(len(self.labels))]
# Generate data
for i, ID in enumerate(list_IDs_temp1):
# Store sample
X1[i,] = np.stack((imageio.imread(ID),) * 3, axis=-1)
# Store class
for list_id in range(len(self.labels)):
y[list_id][i] = self.labels[list_id][indexes[i]]
for i, ID in enumerate(list_IDs_temp2):
X2[i,] = np.stack((imageio.imread(ID),) * 3, axis=-1)
return [X1,X2], y

This way the order of the two batches of data is the same.

[cj00719]

@enviz

Thanks for your guide on multiple inputs, very much appreciated.

I have a similar issue which I used your shared code. However, while executing, I have the TypeError: can't pickle generator objects

Kindly provide guidance to resolve this issue.

Thank you

[cj00719]

@aendrs @enviz
Please, I have the TypeError: can't pickle generator objects

Kindly provide guidance to resolve this issue.

Thank you

[Gijsvo6]

I experience a similar problem.
I wanted to combine two input streams: 1 is an image and 2 is numerical data.

I found that the best solution is to manipulate the keras.utils.Sequence.TimeseriesGenerator functionality for your own purpose here.

For my problem [x1, x2], y this is a working generator:

import numpy as np
import json
import keras

class TimeseriesGenerator(keras.utils.Sequence):

def __init__(self, img, data, targets, length,
             sampling_rate=1,
             stride=1,
             start_index=0,
             end_index=None,
             shuffle=False,
             reverse=False,
             batch_size=128):

    if len(data) != len(targets):
        raise ValueError('Data and targets have to be' +
                         ' of same length. '
                         'Data length is {}'.format(len(data)) +
                         ' while target length is {}'.format(len(targets)))
        
    self.img = img
    self.data = data
    self.targets = targets
    self.length = length
    self.sampling_rate = sampling_rate
    self.stride = stride
    self.start_index = start_index + length
    if end_index is None:
        end_index = len(data) - 1
    self.end_index = end_index
    self.shuffle = shuffle
    self.reverse = reverse
    self.batch_size = batch_size

    if self.start_index > self.end_index:
        raise ValueError('`start_index+length=%i > end_index=%i` '
                         'is disallowed, as no part of the sequence '
                         'would be left to be used as current step.'
                         % (self.start_index, self.end_index))

def __len__(self):
    return (self.end_index - self.start_index +
            self.batch_size * self.stride) // (self.batch_size * self.stride)

def __getitem__(self, index):
    if self.shuffle:
        rows = np.random.randint(
            self.start_index, self.end_index + 1, size=self.batch_size)
    else:
        i = self.start_index + self.batch_size * self.stride * index
        rows = np.arange(i, min(i + self.batch_size *
                                self.stride, self.end_index + 1), self.stride)

    samples2 = np.array([self.img[row - self.length:row:self.sampling_rate]
                        for row in rows])
    samples = np.array([self.data[row - self.length:row:self.sampling_rate]
                        for row in rows])
    targets = np.array([self.targets[row] for row in rows])

    if self.reverse:
        return samples[:, ::-1, ...], targets
    return [samples2, samples], targets

def get_config(self):
    '''Returns the TimeseriesGenerator configuration as Python dictionary.
    # Returns
        A Python dictionary with the TimeseriesGenerator configuration.
    '''
    img = self.img
    if type(self.img).__module__ == np.__name__:
        img = self.img.tolist()
    try:
        json_img = json.dumps(img)
    except TypeError:
        raise TypeError('Data not JSON Serializable:', img)
    
    data = self.data
    if type(self.data).__module__ == np.__name__:
        data = self.data.tolist()
    try:
        json_data = json.dumps(data)
    except TypeError:
        raise TypeError('Data not JSON Serializable:', data)

    targets = self.targets
    if type(self.targets).__module__ == np.__name__:
        targets = self.targets.tolist()
    try:
        json_targets = json.dumps(targets)
    except TypeError:
        raise TypeError('Targets not JSON Serializable:', targets)

    return {
        'img': json_img,
        'data': json_data,
        'targets': json_targets,
        'length': self.length,
        'sampling_rate': self.sampling_rate,
        'stride': self.stride,
        'start_index': self.start_index,
        'end_index': self.end_index,
        'shuffle': self.shuffle,
        'reverse': self.reverse,
        'batch_size': self.batch_size
    }

def to_json(self, **kwargs):
    """Returns a JSON string containing the timeseries generator
    configuration. To load a generator from a JSON string, use
    `keras.preprocessing.sequence.timeseries_generator_from_json(json_string)`.
    # Arguments
        **kwargs: Additional keyword arguments
            to be passed to `json.dumps()`.
    # Returns
        A JSON string containing the tokenizer configuration.
    """
    config = self.get_config()
    timeseries_generator_config = {
        'class_name': self.__class__.__name__,
        'config': config
    }
    return json.dumps(timeseries_generator_config, **kwargs)

def timeseries_generator_from_json(json_string):
"""Parses a JSON timeseries generator configuration file and
returns a timeseries generator instance.
# Arguments
json_string: JSON string encoding a timeseries
generator configuration.
# Returns
A Keras TimeseriesGenerator instance
"""
full_config = json.loads(json_string)
config = full_config.get('config')

img = json.loads(config.pop('img'))
config['img'] = img
data = json.loads(config.pop('data'))
config['data'] = data
targets = json.loads(config.pop('targets'))
config['targets'] = targets

return TimeseriesGenerator(**config)

[sasjafor]

Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful

input_imgen = ImageDataGenerator(rescale = 1./255, 
                                   shear_range = 0.2, 
                                   zoom_range = 0.2,
                                   rotation_range=5.,
                                   horizontal_flip = True)

test_imgen = ImageDataGenerator(rescale = 1./255)



def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
    genX1 = generator.flow_from_directory(dir1,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    
    genX2 = generator.flow_from_directory(dir2,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    while True:
            X1i = genX1.next()
            X2i = genX2.next()
            yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
            
            
inputgenerator=generate_generator_multiple(generator=input_imgen,
                                           dir1=train_dir_1,
                                           dir2=train_dir_2,
                                           batch_size=batch_size,
                                           img_height=img_height,
                                           img_width=img_height)       
     
testgenerator=generate_generator_multiple(test_imgen,
                                          dir1=train_dir_1,
                                          dir2=train_dir_2,
                                          batch_size=batch_size,
                                          img_height=img_height,
                                          img_width=img_height)              
          
 history=model.fit_generator(inputgenerator,
                        steps_per_epoch=trainsetsize/batch_size,
                        epochs = epochs,
                        validation_data = testgenerator,
                        validation_steps = testsetsize/batch_size,
                        use_multiprocessing=True,
                        shuffle=False)

This code is almost perfect, but for me I have the problem that it just runs forever. If I replace the while true I could get it to work as I expected.

[Sayantan2109]

Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful

input_imgen = ImageDataGenerator(rescale = 1./255, 
                                   shear_range = 0.2, 
                                   zoom_range = 0.2,
                                   rotation_range=5.,
                                   horizontal_flip = True)

test_imgen = ImageDataGenerator(rescale = 1./255)



def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
    genX1 = generator.flow_from_directory(dir1,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    
    genX2 = generator.flow_from_directory(dir2,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    while True:
            X1i = genX1.next()
            X2i = genX2.next()
            yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
            
            
inputgenerator=generate_generator_multiple(generator=input_imgen,
                                           dir1=train_dir_1,
                                           dir2=train_dir_2,
                                           batch_size=batch_size,
                                           img_height=img_height,
                                           img_width=img_height)       
     
testgenerator=generate_generator_multiple(test_imgen,
                                          dir1=train_dir_1,
                                          dir2=train_dir_2,
                                          batch_size=batch_size,
                                          img_height=img_height,
                                          img_width=img_height)              
          
 history=model.fit_generator(inputgenerator,
                        steps_per_epoch=trainsetsize/batch_size,
                        epochs = epochs,
                        validation_data = testgenerator,
                        validation_steps = testsetsize/batch_size,
                        use_multiprocessing=True,
                        shuffle=False)

This code is almost perfect, but for me I have the problem that it just runs forever. If I replace the while true I could get it to work as I expected.

in fit.generator..what is trainsetsize and testsetsize???where it is defined??

[gledsonmelotti]

Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful

input_imgen = ImageDataGenerator(rescale = 1./255, 
                                   shear_range = 0.2, 
                                   zoom_range = 0.2,
                                   rotation_range=5.,
                                   horizontal_flip = True)

test_imgen = ImageDataGenerator(rescale = 1./255)



def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
    genX1 = generator.flow_from_directory(dir1,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    
    genX2 = generator.flow_from_directory(dir2,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    while True:
            X1i = genX1.next()
            X2i = genX2.next()
            yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
            
            
inputgenerator=generate_generator_multiple(generator=input_imgen,
                                           dir1=train_dir_1,
                                           dir2=train_dir_2,
                                           batch_size=batch_size,
                                           img_height=img_height,
                                           img_width=img_height)       
     
testgenerator=generate_generator_multiple(test_imgen,
                                          dir1=train_dir_1,
                                          dir2=train_dir_2,
                                          batch_size=batch_size,
                                          img_height=img_height,
                                          img_width=img_height)              
          
 history=model.fit_generator(inputgenerator,
                        steps_per_epoch=trainsetsize/batch_size,
                        epochs = epochs,
                        validation_data = testgenerator,
                        validation_steps = testsetsize/batch_size,
                        use_multiprocessing=True,
                        shuffle=False)

This code is almost perfect, but for me I have the problem that it just runs forever. If I replace the while true I could get it to work as I expected.

in fit.generator..what is trainsetsize and testsetsize???where it is defined??

https://keras.io/api/models/model_training_apis/
https://www.pyimagesearch.com/2018/12/24/how-to-use-keras-fit-and-fit_generator-a-hands-on-tutorial/

[alfredkj7]

yield [X1i[0], X2i[0]]

I am using a similar generator for prediction purposes. But, it is running infinitely. How can I resolve this?

[Morris870820]

Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful

input_imgen = ImageDataGenerator(rescale = 1./255, 
                                   shear_range = 0.2, 
                                   zoom_range = 0.2,
                                   rotation_range=5.,
                                   horizontal_flip = True)

test_imgen = ImageDataGenerator(rescale = 1./255)



def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
    genX1 = generator.flow_from_directory(dir1,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    
    genX2 = generator.flow_from_directory(dir2,
                                          target_size = (img_height,img_width),
                                          class_mode = 'categorical',
                                          batch_size = batch_size,
                                          shuffle=False, 
                                          seed=7)
    while True:
            X1i = genX1.next()
            X2i = genX2.next()
            yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label
            
            
inputgenerator=generate_generator_multiple(generator=input_imgen,
                                           dir1=train_dir_1,
                                           dir2=train_dir_2,
                                           batch_size=batch_size,
                                           img_height=img_height,
                                           img_width=img_height)       
     
testgenerator=generate_generator_multiple(test_imgen,
                                          dir1=train_dir_1,
                                          dir2=train_dir_2,
                                          batch_size=batch_size,
                                          img_height=img_height,
                                          img_width=img_height)              
          
 history=model.fit_generator(inputgenerator,
                        steps_per_epoch=trainsetsize/batch_size,
                        epochs = epochs,
                        validation_data = testgenerator,
                        validation_steps = testsetsize/batch_size,
                        use_multiprocessing=True,
                        shuffle=False)

I have 2 inputs and each one have 5 classes, 30 images. And I use this code but only get

Found 150 images belonging to 5 classes. ---> training data 1 
Found 150 images belonging to 5 classes. ---> validation data 1

But i think it should be like this.

Found 150 images belonging to 5 classes. ---> training data 1 
Found 150 images belonging to 5 classes. ---> training data 2
Found 150 images belonging to 5 classes. ---> validation data 1 
Found 150 images belonging to 5 classes. ---> validation data 2

How can I resolve this?