NameError: name 'x_train' is not defined

[LukeDeWaal]

I know a similar issue is posted in the known issues thread but it does not seem to help me. I keep getting this error no matter what I try, so perhaps I can get answers here. My model is supposed to distinguish between 4 categories of pictures (cats, dogs, airplanes and motorbikes).
I have the following code:

from __future__ import print_function

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os
from keras import layers, models, optimizers
from keras.preprocessing.image import ImageDataGenerator
from hyperas import optim
from hyperopt import Trials, STATUS_OK, tpe
from hyperas.distributions import choice, uniform, conditional
from PIL import Image as im
from os import listdir
from keras.datasets import mnist
from keras.utils import np_utils
from scipy import misc

from keras import backend as K

K.set_image_dim_ordering('tf')


def data():
    
    tag_dict = {'airplane':0, 'cat':1, 'dog':2, 'motorbike':3}
    
    base_directory      = "."
    train_data_dir      = base_directory + "/Train"
    validation_data_dir = base_directory + "/Validation"
    
    maxsize = (100, 100)
    
    trainlen = sum([len(files) for r, d, files in os.walk(train_data_dir)])
    validlen = sum([len(files) for r, d, files in os.walk(validation_data_dir)])
    
    train_array = np.ones((trainlen, 100, 100, 3))
    train_tags  = np.ones((trainlen,))
    
    val_array = np.ones((validlen, 100, 100, 3))
    val_tags  = np.ones((validlen,))
    
    itemcounter = 0
    for folder in listdir(train_data_dir):
        for picture in listdir(train_data_dir + "/" + folder):
            
            tempdirec = train_data_dir + "/" + folder + "/" + picture
            
            img = im.open(tempdirec)
            img = img.resize(maxsize, im.ANTIALIAS)
            
            data = np.asarray(img, dtype='int32')
            
            if data.shape != (100, 100, 3):
                
                data = np.stack((data, data, data), axis=2)
            
            train_array[itemcounter] = data
            train_tags[itemcounter] = tag_dict[folder]
            
            itemcounter += 1
        
    itemcounter = 0
    for folder in listdir(validation_data_dir):
        for picture in listdir(validation_data_dir + "/" + folder):
            
            tempdirec = validation_data_dir + "/" + folder + "/" + picture
            
            img = im.open(tempdirec)
            img = img.resize(maxsize, im.ANTIALIAS)
            
            data = np.asarray(img, dtype='int32')
            
            if data.shape != (100, 100, 3):
                
                data = np.stack((data, data, data), axis=2)
            
            val_array[itemcounter] = data
            val_tags[itemcounter] = tag_dict[folder]
            
            itemcounter += 1
    
    def shuffle(array, seed=16):
        
        np.random.seed(seed)
        np.random.shuffle(array)
        
        return array
    
    nb_classes = 4
    
    train_array,    train_tags   = shuffle(train_array), shuffle(train_tags)
    val_array,      val_tags     = shuffle(val_array),   shuffle(val_tags)  

    train_tags = np_utils.to_categorical(train_tags, nb_classes)     
    val_tags = np_utils.to_categorical(val_tags, nb_classes)
    
    return (train_array, train_tags), (val_array, val_tags)



#%%
def create_model(x_train, y_train, x_test, y_test):
    
    shape = x_train[0].shape
    
    model = models.Sequential()
    model.add(layers.Conv2D(16, (3, 3), activation='tanh', input_shape=shape))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(32, (3, 3), activation='tanh'))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(64, (3, 3), activation='tanh'))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Conv2D(128, (3, 3), activation='tanh'))
    model.add(layers.MaxPooling2D((2, 2)))
    model.add(layers.Flatten())
    model.add(layers.Dropout(rate={{choice([0.2, 0.25, 0.3, 0.35, 0.4])}}))
#    model.add(layers.Dropout(0.25))
    model.add(layers.Dense(128, activation='relu'))
    model.add(layers.Dense(64, activation='relu'))
    model.add(layers.Dense(4, activation='softmax'))
    
    model.compile(loss='categorical_crossentropy', optimizer=optimizers.RMSprop(), metrics=['acc'])
    
    model.summary()
    
    train_generator, validation_generator = data()
    
    history = model.fit(x_train, y_train,         #history = 
                          batch_size = None,
                          steps_per_epoch=5,
                          epochs=4,
                          validation_data=(x_test, y_test),
                          validation_steps=5)

    acc         = history.history['acc']
    val_acc     = history.history['val_acc']
    loss        = history.history['loss']
    val_loss    = history.history['val_loss']
    epochs      = range(1, len(acc) + 1)
    
    
    plt.plot(epochs, acc, 'r-', label='Training acc')
    plt.plot(epochs, val_acc, 'g-', label='Validation acc')
    plt.title('Training and validation accuracy')
    plt.legend()
    plt.figure()
    plt.plot(epochs, loss, 'r-', label='Training loss')
    plt.plot(epochs, val_loss, 'g-', label='Validation loss')
    plt.title('Training and validation loss')
    plt.legend()
    plt.pause(0.05)
    plt.show()
    
    return {'loss': map(lambda x: x * -1.0, acc), 'status': STATUS_OK, 'model': model, 'acc': acc}


if __name__ == '__main__':
    
    best_run, best_model = optim.minimize(model=create_model,
                                          data=data,
                                          algo=tpe.suggest,
                                          max_evals=5,
                                          trials=Trials())
    (x_train, y_train), (x_test, y_test) = data()
    print("Evalutation of best performing model:")
    print(best_model.evaluate(x_test, y_test))
    print("Best performing model chosen hyper-parameters:")
    print(best_run)

And this is the error code:

Traceback (most recent call last):

  File "<ipython-input-17-df9a5e0e117e>", line 1, in <module>
    runfile('C:/Users/lucky/OneDrive/Machine Learning/Crack_Net_5.py', wdir='C:/Users/lucky/OneDrive/Machine Learning')

  File "C:\Users\lucky\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 705, in runfile
    execfile(filename, namespace)

  File "C:\Users\lucky\Anaconda3\lib\site-packages\spyder\utils\site\sitecustomize.py", line 102, in execfile
    exec(compile(f.read(), filename, 'exec'), namespace)

  File "C:/Users/lucky/OneDrive/Machine Learning/Crack_Net_5.py", line 169, in <module>
    trials=Trials())

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperas\optim.py", line 67, in minimize
    verbose=verbose)

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperas\optim.py", line 133, in base_minimizer
    return_argmin=True),

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\fmin.py", line 307, in fmin
    return_argmin=return_argmin,

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\base.py", line 635, in fmin
    return_argmin=return_argmin)

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\fmin.py", line 320, in fmin
    rval.exhaust()

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\fmin.py", line 199, in exhaust
    self.run(self.max_evals - n_done, block_until_done=self.async)

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\fmin.py", line 173, in run
    self.serial_evaluate()

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\fmin.py", line 92, in serial_evaluate
    result = self.domain.evaluate(spec, ctrl)

  File "C:\Users\lucky\Anaconda3\lib\site-packages\hyperopt\base.py", line 840, in evaluate
    rval = self.fn(pyll_rval)

  File "C:\Users\lucky\OneDrive\Machine Learning\temp_model.py", line 160, in keras_fmin_fnct

NameError: name 'x_train' is not defined

If anyone could help me out that'd be great, I've been breaking my head over this for multiple days now.

[maxpumperla]

@LukeDeWaal it's simply that the return values of data have to exactly match the input to model, due to questionable design choices on my part. :) So, this:

...
return (train_array, train_tags), (val_array, val_tags)

def create_model(x_train, y_train, x_test, y_test):
    ...

simply won't do. x_train isn't available to hyperas this way.

[sanesanyo]

I am still getting the same error even when I have exactly matched the return value of the 'data' function with the input values to the 'create_model' function.
As a matter of fact, when I run the code from 'hyperas/examples/mnist_readme.py', I still get the same error.
I would be extremely thankful if you could me some tips regarding what could be going wrong.

[maxpumperla]

@sanesanyo do me a favor and do the following for me:

git clone https://github.com/maxpumperla/hyperas
cd hyperas
virtualenv venv
source venv/bin/activate
pip install tensorflow
python setup.py install
python examples/mnist_readme.py

If that doesn't work for you I'd be very surprised. Provide me with the full stack trace in that case.

[sanesanyo]

Hi Max, First of all, thank you for your reply. This seemed to work. What do you think could be the reason why it didn't work before? Regards, Sanyog

…

On Sun, Sep 2, 2018, 9:30 PM Max Pumperla ***@***.***> wrote: @sanesanyo <https://github.com/sanesanyo> do me a favor and do the following for me: git clone https://github.com/maxpumperla/hyperas cd hyperas virtualenv venv source venv/bin/activate pip install tensorflow python setup.py install python examples/mnist_readme.py If that doesn't work for you I'd be very surprised. Provide me with the full stack trace in that case. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <#168 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AVY-AUIFkszR0aUreJaZg5mxtzoJQfxPks5uXDHRgaJpZM4UDTYr> .

[maxpumperla]

maybe storing but not deleting the temp file? in any case, use virtual envs!

[slundberg]

For what it is worth I had the same error and deleting the __pycache__ directory fixed the problem (the one created in the same directory as the script or notebook).

[maxpumperla]

good point. I can't justify to delete the pycache after each hyperas run, though. so I'm afraid this is something users have to deal with

[sanesanyo]

Deleting the cache worked for me as well. Thanks for the help once again. Just one final question, is there any documentation somewhere which explains the output of the best model. I understand most of the output but am getting confused with 'add' key. I would be extremely grateful if you could throw some light on it. I did find some links on the web but unfortunately couldn't find the answers. Regards

…

On Thu, Sep 6, 2018, 8:36 AM Max Pumperla ***@***.***> wrote: good point. I can't justify to delete the pycache after each hyperas run, though. so I'm afraid this is something users have to deal with — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <#168 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AVY-AW_c3WTuMRaSOaURyfR28gAOuNVUks5uYMJigaJpZM4UDTYr> .

[mayuriamale]

NameError Traceback (most recent call last)
in ()
8 x2,y2 = int(box[2]-w),int(box[3]+h)
9 return abs(x1), abs(y1), abs(x2), abs(y2)
---> 10 train.head()
11 boxes = [[361.0,624.0,98.0,437.0]]
12 img1=cv2.imread('/content/All_Images/0_Parade_marchingband_1_300.jpg')

NameError: name 'train' is not defined
error related to name 'train' is not defined plzz help me to solve this query
Duplicate of #

[rezapci]

still no help on NameError: name 'train' is not defined ? anyone solved this issue?