ValueError: I/O operation on closed file

[angy50]

I am trying to run the following code in anaconda on 64 bit windows 8.1. But it is generating error randomly after some epochs. Please help to get out of this.

Code:

from future import print_function
import os
import datetime
import time
import numpy as np
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD,Adam, RMSprop
from keras.utils import np_utils
import matplotlib.pyplot as plt
from keras.callbacks import Callback, ModelCheckpoint, History

batch_size = 32
nb_classes = 10
nb_epoch = 50

nwt = datetime.datetime.now

input image dimensions

img_rows, img_cols = 32, 32

the CIFAR10 images are RGB

img_channels = 3

the data, shuffled and split between train and test sets

(X_train, y_train), (X_test, y_test) = cifar10.load_data()

X_train = X_train[0:1000,:]

y_train = y_train[0:1000]

X_test = X_test[0:1000,:]

y_test = y_test[0:1000,]

print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')

convert class vectors to binary class matrices

Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)

model = Sequential()

model.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))

let's train the model using SGD + momentum (how original).

sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics = ['accuracy'])

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

start_time = nwt()

print('Using real-time data augmentation.')
WEIGHTS_FNAME = 'cifar_aug_cnn_weights.hdf'

if False and os.path.exists(WEIGHTS_FNAME):
# Just change the True to false to force re-training
print('Loading existing weights')
model.load_weights(WEIGHTS_FNAME)
else:
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=False) # randomly flip images

# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)

# fit the model on the batches generated by datagen.flow()

checkpt = ModelCheckpoint(filepath="F:\own_trial\p1_da\data\weights_{epoch:}.hdf", verbose=0, save_best_only=False)

model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
                        samples_per_epoch=X_train.shape[0],
                        nb_epoch=nb_epoch, verbose = 1,
                        validation_data=(X_test, Y_test),
                        nb_worker=1)

time.sleep(3)

model.save_weights(WEIGHTS_FNAME)

---

Error occured at 43th epoch after processing 3424 files is:
Epoch 43/50
3424/50000 [=>............................] - ETA: 91s - loss: 0.7680 - acc: 0.7314 Traceback (most recent call last):

File "", line 1, in
runfile('F:/da_trail/p1_da_50/demo_da.py', wdir='F:/da_trail/p1_da_50')

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 714, in runfile
execfile(filename, namespace)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\spyderlib\widgets\externalshell\sitecustomize.py", line 74, in execfile
exec(compile(scripttext, filename, 'exec'), glob, loc)

File "F:/da_trail/p1_da_50/demo_da.py", line 116, in
nb_worker=1)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\keras\models.py", line 851, in fit_generator
pickle_safe=pickle_safe)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\keras\engine\training.py", line 1454, in fit_generator
callbacks.on_batch_end(batch_index, batch_logs)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\keras\callbacks.py", line 61, in on_batch_end
callback.on_batch_end(batch, logs)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\keras\callbacks.py", line 189, in on_batch_end
self.progbar.update(self.seen, self.log_values)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\keras\utils\generic_utils.py", line 119, in update
sys.stdout.write(info)

File "C:\Users\Y50-70\Anaconda2\lib\site-packages\ipykernel\iostream.py", line 317, in write
self._buffer.write(string)

ValueError: I/O operation on closed file

[jaikumarm]

duplicate of #2110, see ipython/ipython#9168 for the original issue, possibly fixed in ipython 4.4 (not released yet)

[angy50]

I have tried that time.sleep() trick but it doesn't worked for my case.
Keeping verbose value = 2 has worked. I am not able to see the progress bar in that case but its still a better option. No error is observed with verbose=2.
Thanks.

[shachar-i]

verbose=2 worked for me. Thanks @angy50 !