DeepLab V3 + predict a black mask

[ouzzane]

I trained the DeepLab V3 + network on my own dataset, but after the test I had black images after ten iteration of the training.
then I did several tests, where I took into account that the training was working fine at the start but apartere from [6-10] training was starting to diverge towards a black image.
the source code: https://github.com/rishizek/tensorflow-deeplab-v3-plus
so i want to know where the problem is coming from, can you help me please?

[ravikyram]

@ouzzane

Please, fill issue template..

Can you please share colab link or code snippet to reproduce the issue in our environment.It helps us in localizing the issue faster.Thanks!

[ouzzane]

#train
from future import absolute_import
from future import division
from future import print_function

import argparse
import os
import sys

import tensorflow as tf
import deeplab_model
from utils import preprocessing
from tensorflow.python import debug as tf_debug
import matplotlib.pyplot as plt
import shutil
parser = argparse.ArgumentParser()

parser.add_argument('--model_dir', type=str, default='./model/',
help='Base directory for the model.')

parser.add_argument('--clean_model_dir', action='store_true',
help='Whether to clean up the model directory if present.')

parser.add_argument('--train_epochs', type=int, default=26,
help='Number of training epochs: '
'For 30K iteration with batch size 6, train_epoch = 17.01 (= 30K * 6 / 10,582). '
'For 30K iteration with batch size 8, train_epoch = 22.68 (= 30K * 8 / 10,582). '
'For 30K iteration with batch size 10, train_epoch = 25.52 (= 30K * 10 / 10,582). '
'For 30K iteration with batch size 11, train_epoch = 31.19 (= 30K * 11 / 10,582). '
'For 30K iteration with batch size 15, train_epoch = 42.53 (= 30K * 15 / 10,582). '
'For 30K iteration with batch size 16, train_epoch = 45.36 (= 30K * 16 / 10,582).')

parser.add_argument('--epochs_per_eval', type=int, default=1,
help='The number of training epochs to run between evaluations.')

parser.add_argument('--tensorboard_images_max_outputs', type=int, default=6,
help='Max number of batch elements to generate for Tensorboard.')

parser.add_argument('--batch_size', type=int, default=2,
help='Number of examples per batch.')

parser.add_argument('--learning_rate_policy', type=str, default='poly',
choices=['poly', 'piecewise'],
help='Learning rate policy to optimize loss.')

parser.add_argument('--max_iter', type=int, default=60000,
help='Number of maximum iteration used for "poly" learning rate policy.')

parser.add_argument('--data_dir', type=str, default='./dataset/',
help='Path to the directory containing the PASCAL VOC data tf record.')

parser.add_argument('--base_architecture', type=str, default='resnet_v2_101',
choices=['resnet_v2_50', 'resnet_v2_101'],
help='The architecture of base Resnet building block.')

parser.add_argument('--pre_trained_model', type=str, default='./ini_checkpoints/resnet_v2_101/resnet_v2_101.ckpt',
help='Path to the pre-trained model checkpoint.')

parser.add_argument('--output_stride', type=int, default=16,
choices=[8, 16],
help='Output stride for DeepLab v3. Currently 8 or 16 is supported.')

parser.add_argument('--freeze_batch_norm', action='store_true',
help='Freeze batch normalization parameters during the training.')

parser.add_argument('--initial_learning_rate', type=float, default=0.01 ,
help='Initial learning rate for the optimizer.')

parser.add_argument('--end_learning_rate', type=float, default=1e-4,
help='End learning rate for the optimizer.')

parser.add_argument('--initial_global_step', type=int, default=0,
help='Initial global step for controlling learning rate when fine-tuning model.')

parser.add_argument('--weight_decay', type=float, default=2e-4,
help='The weight decay to use for regularizing the model.')

parser.add_argument('--debug', action='store_true',
help='Whether to use debugger to track down bad values during training.')

_NUM_CLASSES = 6
_HEIGHT =576
_WIDTH = 432
_DEPTH = 3
_MIN_SCALE = 0.5
_MAX_SCALE = 2.0
_IGNORE_LABEL = 255

_POWER = 0.9
_MOMENTUM = 0.9

_BATCH_NORM_DECAY = 0.9997

_NUM_IMAGES = {
'train': 1008,
'validation': 120,
}

def get_filenames(is_training, data_dir):
"""Return a list of filenames.

Args:
is_training: A boolean denoting whether the input is for training.
data_dir: path to the the directory containing the input data.

Returns:
A list of file names.
"""
if is_training:
return [os.path.join(data_dir, 'PQR/tfrecord/PQR_train.record')]
else:
return [os.path.join(data_dir, 'PQR/tfrecord/PQR_val.record')]

def parse_record(raw_record):
"""Parse PASCAL image and label from a tf record."""
keys_to_features = {
'image/height':
tf.FixedLenFeature((), tf.int64),
'image/width':
tf.FixedLenFeature((), tf.int64),
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpeg'),
'label/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'label/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
}

parsed = tf.parse_single_example(raw_record, keys_to_features)

height = tf.cast(parsed['image/height'], tf.int32)

width = tf.cast(parsed['image/width'], tf.int32)

image = tf.image.decode_image(
tf.reshape(parsed['image/encoded'], shape=[]), _DEPTH)
image = tf.to_float(tf.image.convert_image_dtype(image, dtype=tf.uint8))
image.set_shape([None, None, 3])

label = tf.image.decode_image(
tf.reshape(parsed['label/encoded'], shape=[]), 1)
label = tf.to_int32(tf.image.convert_image_dtype(label, dtype=tf.uint8))
label.set_shape([None, None, 1])

return image, label

def preprocess_image(image, label, is_training):
"""Preprocess a single image of layout [height, width, depth]."""
if is_training:
# Randomly scale the image and label.
image, label = preprocessing.random_rescale_image_and_label(
image, label, _MIN_SCALE, _MAX_SCALE)

# Randomly crop or pad a [_HEIGHT, _WIDTH] section of the image and label.
image, label = preprocessing.random_crop_or_pad_image_and_label(
    image, label, _HEIGHT, _WIDTH, _IGNORE_LABEL)

# Randomly flip the image and label horizontally.
image, label = preprocessing.random_flip_left_right_image_and_label(
    image, label)

image.set_shape([_HEIGHT, _WIDTH, 3])
label.set_shape([_HEIGHT, _WIDTH, 1])

image = preprocessing.mean_image_subtraction(image)

return image, label

def input_fn(is_training, data_dir, batch_size, num_epochs=1):
"""Input_fn using the tf.data input pipeline for CIFAR-10 dataset.

Args:
is_training: A boolean denoting whether the input is for training.
data_dir: The directory containing the input data.
batch_size: The number of samples per batch.
num_epochs: The number of epochs to repeat the dataset.

Returns:
A tuple of images and labels.
"""
dataset = tf.data.Dataset.from_tensor_slices(get_filenames(is_training, data_dir))
dataset = dataset.flat_map(tf.data.TFRecordDataset)

if is_training:
# When choosing shuffle buffer sizes, larger sizes result in better
# randomness, while smaller sizes have better performance.
# is a relatively small dataset, we choose to shuffle the full epoch.
dataset = dataset.shuffle(buffer_size=_NUM_IMAGES['train'])

dataset = dataset.map(parse_record)
dataset = dataset.map(
lambda image, label: preprocess_image(image, label, is_training))
dataset = dataset.prefetch(batch_size)

We call repeat after shuffling, rather than before, to prevent separate

epochs from blending together.

dataset = dataset.repeat(num_epochs)
dataset = dataset.batch(batch_size)

iterator = dataset.make_one_shot_iterator()
images, labels = iterator.get_next()

return images, labels

def main(unused_argv):

Using the Winograd non-fused algorithms provides a small performance boost.

os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'

if FLAGS.clean_model_dir:
shutil.rmtree(FLAGS.model_dir, ignore_errors=True)

Set up a RunConfig to only save checkpoints once per training cycle.

run_config = tf.estimator.RunConfig().replace(save_checkpoints_secs=1e9)
model = tf.estimator.Estimator(
model_fn=deeplab_model.deeplabv3_plus_model_fn,
model_dir=FLAGS.model_dir,
config=run_config,
params={
'output_stride': FLAGS.output_stride,
'batch_size': FLAGS.batch_size,
'base_architecture': FLAGS.base_architecture,
'pre_trained_model': FLAGS.pre_trained_model,
'batch_norm_decay': _BATCH_NORM_DECAY,
'num_classes': _NUM_CLASSES,
'tensorboard_images_max_outputs': FLAGS.tensorboard_images_max_outputs,
'weight_decay': FLAGS.weight_decay,
'learning_rate_policy': FLAGS.learning_rate_policy,
'num_train': _NUM_IMAGES['train'],
'initial_learning_rate': FLAGS.initial_learning_rate,
'max_iter': FLAGS.max_iter,
'end_learning_rate': FLAGS.end_learning_rate,
'power': _POWER,
'momentum': _MOMENTUM,
'freeze_batch_norm': FLAGS.freeze_batch_norm,
'initial_global_step': FLAGS.initial_global_step
})

for _ in range(FLAGS.train_epochs // FLAGS.epochs_per_eval):
tensors_to_log = {
'learning_rate': 'learning_rate',
'cross_entropy': 'cross_entropy',
'train_px_accuracy': 'train_px_accuracy',
'train_mean_iou': 'train_mean_iou',
}

logging_hook = tf.train.LoggingTensorHook(
    tensors=tensors_to_log, every_n_iter=10)
train_hooks = [logging_hook]
eval_hooks = None

if FLAGS.debug:
  debug_hook = tf_debug.LocalCLIDebugHook()
  train_hooks.append(debug_hook)
  eval_hooks = [debug_hook]

tf.logging.info("Start training.")
model.train(
    input_fn=lambda: input_fn(True, FLAGS.data_dir, FLAGS.batch_size, FLAGS.epochs_per_eval),
    hooks=train_hooks,
    # steps=1  # For debug
)

tf.logging.info("Start evaluation.")
# Evaluate the model and print results
eval_results = model.evaluate(
    # Batch size must be 1 for testing because the images' size differs
    input_fn=lambda: input_fn(False, FLAGS.data_dir, 1),
    hooks=eval_hooks,
    # steps=1  # For debug
)
print(eval_results)

if name == 'main':
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
FLAGS, unparsed = parser.parse_known_args()
tf.compat.v1.app.run(main=main, argv=[sys.argv[0]] + unparsed)

[ravikyram]

@ouzzane

Which tensorflow version you are using?

Thanks!

[ouzzane]

Hello,
thank you for your reply ravikyram
I am using tensorflow 1.15
Best regards