From 8ce65f4071e00c8ca50c7e38d909a7ff67b8f552 Mon Sep 17 00:00:00 2001 From: willduan Date: Mon, 19 Jun 2017 00:50:52 +0800 Subject: [PATCH 01/61] add contrastive loss (#801) * add contrastive loss * add contrastive loss --- tflearn/objectives.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tflearn/objectives.py b/tflearn/objectives.py index b938ea2b..967a4ed4 100644 --- a/tflearn/objectives.py +++ b/tflearn/objectives.py @@ -256,3 +256,24 @@ def weak_cross_entropy_2d(y_pred, y_true, num_classes=None, epsilon=0.0001, name="xentropy_mean") return cross_entropy_mean + +def contrastive_loss(y_pred, y_true, margin = 1.0): + """ Contrastive Loss. + + Computes the constrative loss between y_pred (logits) and + y_true (labels). + + http://yann.lecun.com/exdb/publis/pdf/chopra-05.pdf + Sumit Chopra, Raia Hadsell and Yann LeCun (2005). + Learning a Similarity Metric Discriminatively, with Application to Face Verification. + + Arguments: + y_pred: `Tensor`. Predicted values. + y_true: `Tensor`. Targets (labels). + margin: . A self-set parameters that indicate the distance between the expected different identity features. Defaults 1. + """ + + with tf.name_scope("ContrastiveLoss"): + dis1 = y_true * tf.square(y_pred) + dis2 = (1 - y_true) * tf.square(tf.maximum((margin - y_pred), 0)) + return tf.reduce_sum(dis1 +dis2) / 2. From 8af77b5aebcb8aba0f1b855201aed732906c6de8 Mon Sep 17 00:00:00 2001 From: Eduardo Pinho Date: Thu, 22 Jun 2017 17:47:18 +0100 Subject: [PATCH 02/61] Fix tflearn.activation on callable object (#805) --- tests/test.py | 30 ++++++++++++++++++++++++++++++ tflearn/layers/core.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 3daf305a..ae066dbe 100644 --- a/tests/test.py +++ b/tests/test.py @@ -59,5 +59,35 @@ def test_leaky_relu(self): self.assertAlmostEqual(sess.run(f(x), feed_dict={x:-5}), -1, places=TestActivations.PLACES) + def test_apply_activation(self): + lrelu_02 = lambda x: tflearn.leaky_relu(x, alpha=0.2) + x = tf.constant(-0.25, tf.float32) + + with tf.Session() as sess: + # Case 1: 'linear' + self.assertEqual( + sess.run(tflearn.activation(x, 'linear')), + -0.25) + + # Case 2: 'relu' + self.assertEqual( + sess.run(tflearn.activation(x, 'relu')), + 0) + + # Case 3: 'leaky_relu' + self.assertAlmostEqual( + sess.run(tflearn.activation(x, 'leaky_relu')), + -0.025, places=TestActivations.PLACES) + + # Case 4: 'tanh' + self.assertAlmostEqual( + sess.run(tflearn.activation(x, 'tanh')), + -0.2449, places=TestActivations.PLACES) + + # Case 5: lrelu_02 (callable) + self.assertAlmostEqual( + sess.run(tflearn.activation(x, lrelu_02)), + -0.05, places=TestActivations.PLACES) + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tflearn/layers/core.py b/tflearn/layers/core.py index c180535d..f1dfbd41 100644 --- a/tflearn/layers/core.py +++ b/tflearn/layers/core.py @@ -338,7 +338,7 @@ def activation(incoming, activation='linear', name='activation'): if isinstance(activation, str): x = activations.get(activation)(incoming) - elif hasattr(incoming, '__call__'): + elif hasattr(activation, '__call__'): x = activation(incoming) else: raise ValueError('Unknown activation type.') From 912cdc3327d4de543e16f0af03546756132b7a66 Mon Sep 17 00:00:00 2001 From: Jesper Jaxing Date: Thu, 29 Jun 2017 02:05:34 +0200 Subject: [PATCH 03/61] fixed typo (#807) --- docs/templates/getting_started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/templates/getting_started.md b/docs/templates/getting_started.md index adf05632..0a011ec4 100644 --- a/docs/templates/getting_started.md +++ b/docs/templates/getting_started.md @@ -45,7 +45,7 @@ File | Layers ### Built-in Operations -Besides layers concept, TFLearn also provides many different ops to be used when building a neural network. These ops are firstly mean to be part of the above 'layers' arguments, but they can also be used independently in any other Tensorflow graph for convenience. In practice, just providing the op name as argument is enough (such as activation='relu' or regularizer='L2' for conv_2d), but a function can also be provided for further customization. +Besides layers concept, TFLearn also provides many different ops to be used when building a neural network. These ops are firstly meant to be part of the above 'layers' arguments, but they can also be used independently in any other Tensorflow graph for convenience. In practice, just providing the op name as argument is enough (such as activation='relu' or regularizer='L2' for conv_2d), but a function can also be provided for further customization. File | Ops -----|---- From d93a32ecdadeff89f13a67d0a8fdb7d08cef9035 Mon Sep 17 00:00:00 2001 From: Ruslan Melnikov Date: Thu, 29 Jun 2017 09:09:55 +0300 Subject: [PATCH 04/61] L2 nofm fix for tensorflow 1.0 (#810) --- tflearn/layers/normalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/layers/normalization.py b/tflearn/layers/normalization.py index dd66011e..7e36b692 100644 --- a/tflearn/layers/normalization.py +++ b/tflearn/layers/normalization.py @@ -177,7 +177,7 @@ def l2_normalize(incoming, dim, epsilon=1e-12, name="l2_normalize"): A `Tensor` with the same shape as `x`. """ with tf.name_scope(name) as name: - x = tf.ops.convert_to_tensor(incoming, name="x") + x = tf.convert_to_tensor(incoming, name="x") square_sum = tf.reduce_sum(tf.square(x), [dim], keep_dims=True) x_inv_norm = tf.rsqrt(tf.maximum(square_sum, epsilon)) From 644ecd46b2a9fa7731f7da9218f77a641efcbd21 Mon Sep 17 00:00:00 2001 From: ArtiPyHeart <602416223@qq.com> Date: Sat, 8 Jul 2017 07:10:26 +0800 Subject: [PATCH 05/61] Fix function to_categorical in data_utils.py (#816) The asarray function of numpy will keep ndarray's shape, so the original to_categorical function cannot return the right answer when y is a ndarray with 2-dimension or more. A reshape function is added for changing ndarray to 1-d array. And add a warning when the input array is 3-dimension or more. --- tflearn/data_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index fd95047a..65ad2118 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -7,6 +7,7 @@ from PIL import Image import pickle import csv +import warnings """ Preprocessing provides some useful functions to preprocess data before @@ -36,6 +37,12 @@ def to_categorical(y, nb_classes): """ y = np.asarray(y, dtype='int32') + # high dimensional array warning + if len(y.shape) > 2: + warnings.warn('{}-dimensional array is used as input array.'.format(len(y.shape)), stacklevel=2) + # flatten high dimensional array + if len(y.shape) > 1: + y = y.reshape(-1) if not nb_classes: nb_classes = np.max(y)+1 Y = np.zeros((len(y), nb_classes)) From d01a0b9f59eb8da2e5db9f5fc64617c6379d1ab2 Mon Sep 17 00:00:00 2001 From: Colin Skow Date: Mon, 10 Jul 2017 05:39:53 -0700 Subject: [PATCH 06/61] Fix `bidirectional_rnn` with TF >= 1.2 (resolves #818) (#825) * fix bidirectional_rnn working with TF 1.2 (resolves #818) * fix bidirectional RNN, ensure backward compatibility --- tflearn/layers/recurrent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/layers/recurrent.py b/tflearn/layers/recurrent.py index 0d4e25c2..99d67d8f 100644 --- a/tflearn/layers/recurrent.py +++ b/tflearn/layers/recurrent.py @@ -6,7 +6,7 @@ import tensorflow as tf from tensorflow.python.ops import array_ops try: - from tensorflow.python.ops.rnn import rnn_cell_impl as _rnn_cell, dynamic_rnn as _drnn, static_rnn as _rnn, bidirectional_dynamic_rnn as _brnn + from tensorflow.python.ops.rnn import rnn_cell_impl as _rnn_cell, dynamic_rnn as _drnn, static_rnn as _rnn, static_bidirectional_rnn as _brnn core_rnn_cell = _rnn_cell except: # Fix for TF 1.1.0 and under From 6fd2a9dde347189c08e5f13feb3d544a7857f8a4 Mon Sep 17 00:00:00 2001 From: willduan Date: Wed, 12 Jul 2017 21:27:33 +0800 Subject: [PATCH 07/61] Fix "using a `tf.Tensor` as a Python `bool` is not allowed" (#820) * add contrastive loss * add contrastive loss * Fix Using a as a Python is not allowed --- tflearn/layers/conv.py | 32 ++++++++++++++++---------------- tflearn/layers/core.py | 8 ++++---- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index 2843e139..b1e2522f 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -78,7 +78,7 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -97,7 +97,7 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) inference = tf.nn.conv2d(incoming, W, strides, padding) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if activation: if isinstance(activation, str): @@ -196,7 +196,7 @@ def conv_2d_transpose(incoming, nb_filter, filter_size, output_shape, if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, @@ -229,7 +229,7 @@ def conv_2d_transpose(incoming, nb_filter, filter_size, output_shape, # Reshape tensor so its shape is correct. inference.set_shape([None] + output_shape) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if isinstance(activation, str): inference = activations.get(activation)(inference) @@ -345,7 +345,7 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -364,7 +364,7 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) inference = tf.nn.atrous_conv2d(incoming, W, rate, padding) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if activation: if isinstance(activation, str): @@ -475,7 +475,7 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -494,7 +494,7 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) inference = tf.nn.depthwise_conv2d(incoming, W, strides, padding) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if activation: if isinstance(activation, str): @@ -818,7 +818,7 @@ def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -838,7 +838,7 @@ def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', # Adding dummy dimension to fit with Tensorflow conv2d inference = tf.expand_dims(incoming, 2) inference = tf.nn.conv2d(inference, W, strides, padding) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) inference = tf.squeeze(inference, [2]) if isinstance(activation, str): @@ -1023,7 +1023,7 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -1041,7 +1041,7 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) inference = tf.nn.conv3d(incoming, W, strides, padding) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if isinstance(activation, str): inference = activations.get(activation)(inference) @@ -1140,7 +1140,7 @@ def conv_3d_transpose(incoming, nb_filter, filter_size, output_shape, if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, @@ -1173,7 +1173,7 @@ def conv_3d_transpose(incoming, nb_filter, filter_size, output_shape, # Reshape tensor so its shape is correct. inference.set_shape([None] + output_shape) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if isinstance(activation, str): inference = activations.get(activation)(inference) @@ -1760,7 +1760,7 @@ def highway_conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -1887,7 +1887,7 @@ def highway_conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, diff --git a/tflearn/layers/core.py b/tflearn/layers/core.py index f1dfbd41..46d8df83 100644 --- a/tflearn/layers/core.py +++ b/tflearn/layers/core.py @@ -150,7 +150,7 @@ def fully_connected(incoming, n_units, activation='linear', bias=True, if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = va.variable('W', shape=[n_inputs, n_units], regularizer=W_regul, initializer=W_init, trainable=trainable, @@ -171,7 +171,7 @@ def fully_connected(incoming, n_units, activation='linear', bias=True, inference = tf.reshape(inference, [-1, n_inputs]) inference = tf.matmul(inference, W) - if b: inference = tf.nn.bias_add(inference, b) + if b is not None: inference = tf.nn.bias_add(inference, b) if activation: if isinstance(activation, str): inference = activations.get(activation)(inference) @@ -407,7 +407,7 @@ def single_unit(incoming, activation='linear', bias=True, trainable=True, inference = tf.reshape(inference, [-1]) inference = tf.multiply(inference, W) - if b: inference = tf.add(inference, b) + if b is not None: inference = tf.add(inference, b) if isinstance(activation, str): inference = activations.get(activation)(inference) @@ -494,7 +494,7 @@ def highway(incoming, n_units, activation='linear', transform_dropout=None, if isinstance(weights_init, str): W_init = initializations.get(weights_init)() W_regul = None - if regularizer: + if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) W = va.variable('W', shape=[n_inputs, n_units], regularizer=W_regul, initializer=W_init, trainable=trainable, From b52030a1fc8ce7b0a63be3c0882a1a36cbdd0311 Mon Sep 17 00:00:00 2001 From: exelents Date: Thu, 13 Jul 2017 04:05:56 +1000 Subject: [PATCH 08/61] Fix problem with excluding training ops (#835) --- tflearn/helpers/trainer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tflearn/helpers/trainer.py b/tflearn/helpers/trainer.py index 689c4625..91d41789 100644 --- a/tflearn/helpers/trainer.py +++ b/tflearn/helpers/trainer.py @@ -237,10 +237,9 @@ def fit(self, feed_dicts, n_epoch=10, val_feed_dicts=None, show_metric=False, original_train_ops = list(self.train_ops) # Remove excluded train_ops - for t in self.train_ops: - if excl_trainops and t in excl_trainops: - self.train_ops.remove(t) - + if excl_trainops: + self.train_ops = list(filter(lambda a: a not in excl_trainops, self.train_ops)) + # shuffle is an override for simplicty, it will overrides every # training op batch shuffling if isinstance(shuffle_all, bool): From af57b1759c0d251313c5bcde8cbb7274bf4b08c3 Mon Sep 17 00:00:00 2001 From: Shuolongbj Date: Sat, 15 Jul 2017 06:59:02 +0800 Subject: [PATCH 09/61] update:recommender_wide_and_deep.py (#838) tensorflow:tf.variable_op_scope(values, name, default_name) is deprecated, use tf.variable_scope(name, default_name, values) --- examples/others/recommender_wide_and_deep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/others/recommender_wide_and_deep.py b/examples/others/recommender_wide_and_deep.py index f51262dc..5429fc8b 100644 --- a/examples/others/recommender_wide_and_deep.py +++ b/examples/others/recommender_wide_and_deep.py @@ -115,7 +115,7 @@ def build_model(self, learning_rate=[0.001, 0.01]): with tf.name_scope("Y"): # placeholder for target variable (i.e. trainY input) Y_in = tf.placeholder(shape=[None, 1], dtype=tf.float32, name="Y") - with tf.variable_op_scope([wide_inputs], None, "cb_unit", reuse=False) as scope: + with tf.variable_scope(None, "cb_unit", [wide_inputs]) as scope: central_bias = tflearn.variables.variable('central_bias', shape=[1], initializer=tf.constant_initializer(np.random.randn()), trainable=True, restore=True) From 9d48f42c63f629dd65202b9666b1dfdf5a64bda6 Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Tue, 25 Jul 2017 15:25:46 +0100 Subject: [PATCH 10/61] add densenet layer & example --- docs/templates/examples.md | 4 +- examples/README.md | 3 +- examples/images/densenet.py | 65 ++++++++++++++++++ tflearn/__init__.py | 2 +- tflearn/layers/__init__.py | 2 +- tflearn/layers/conv.py | 127 ++++++++++++++++++++++++++++++++++++ 6 files changed, 199 insertions(+), 4 deletions(-) create mode 100644 examples/images/densenet.py diff --git a/docs/templates/examples.md b/docs/templates/examples.md index 992100f8..896ec457 100644 --- a/docs/templates/examples.md +++ b/docs/templates/examples.md @@ -29,8 +29,10 @@ - [Highway Convolutional Network](https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_highway_mnist.py). Highway Convolutional Network implementation for classifying MNIST dataset. - [Residual Network (MNIST)](https://github.com/tflearn/tflearn/blob/master/examples/images/residual_network_mnist.py). A bottleneck residual network applied to MNIST classification task. - [Residual Network (CIFAR-10)](https://github.com/tflearn/tflearn/blob/master/examples/images/residual_network_cifar10.py). A residual network applied to CIFAR-10 classification task. -- [ResNeXt (CIFAR-10)](https://github.com/tflearn/tflearn/blob/master/examples/images/resnext_cifar10.py). Aggregated residual transformations network (ResNeXt) applied to CIFAR-10 classification task. +- [ResNeXt](https://github.com/tflearn/tflearn/blob/master/examples/images/resnext_cifar10.py). Aggregated residual transformations network (ResNeXt) applied to CIFAR-10 classification task. +- [DenseNet](https://github.com/tflearn/tflearn/blob/master/examples/images/densenet.py). A densely connected convolutional network applied to CIFAR-10 classification task. - [Google Inception (v3)](https://github.com/tflearn/tflearn/blob/master/examples/images/googlenet.py). Google's Inception v3 network applied to Oxford Flowers 17 classification task. + ### Unsupervised - [Auto Encoder](https://github.com/tflearn/tflearn/blob/master/examples/images/autoencoder.py). An auto encoder applied to MNIST handwritten digits. - [Variational Auto Encoder](https://github.com/tflearn/tflearn/blob/master/examples/images/variational_autoencoder.py). A Variational Auto Encoder (VAE) trained to generate digit images. diff --git a/examples/README.md b/examples/README.md index a2f34878..fd7a15c0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -29,7 +29,8 @@ - [Highway Convolutional Network](https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_highway_mnist.py). Highway Convolutional Network implementation for classifying MNIST dataset. - [Residual Network (MNIST)](https://github.com/tflearn/tflearn/blob/master/examples/images/residual_network_mnist.py). A bottleneck residual network applied to MNIST classification task. - [Residual Network (CIFAR-10)](https://github.com/tflearn/tflearn/blob/master/examples/images/residual_network_cifar10.py). A residual network applied to CIFAR-10 classification task. -- [ResNeXt (CIFAR-10)](https://github.com/tflearn/tflearn/blob/master/examples/images/resnext_cifar10.py). Aggregated residual transformations network (ResNeXt) applied to CIFAR-10 classification task. +- [ResNeXt](https://github.com/tflearn/tflearn/blob/master/examples/images/resnext_cifar10.py). Aggregated residual transformations network (ResNeXt) applied to CIFAR-10 classification task. +- [DenseNet](https://github.com/tflearn/tflearn/blob/master/examples/images/densenet.py). A densely connected convolutional network applied to CIFAR-10 classification task. - [Google Inception (v3)](https://github.com/tflearn/tflearn/blob/master/examples/images/googlenet.py). Google's Inception v3 network applied to Oxford Flowers 17 classification task. ### Unsupervised - [Auto Encoder](https://github.com/tflearn/tflearn/blob/master/examples/images/autoencoder.py). An auto encoder applied to MNIST handwritten digits. diff --git a/examples/images/densenet.py b/examples/images/densenet.py new file mode 100644 index 00000000..afb09db9 --- /dev/null +++ b/examples/images/densenet.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +""" Densely Connected Convolutional Networks. + +Applying a 'DenseNet' to CIFAR-10 Dataset classification task. + +References: + - G. Huang, Z. Liu, K. Q. Weinberger, L. van der Maaten. Densely Connected + Convolutional Networks, 2016. + +Links: + - [Densely Connected Convolutional Networks](https://arxiv.org/abs/1608.06993) + - [CIFAR-10 Dataset](https://www.cs.toronto.edu/~kriz/cifar.html) + +""" + +from __future__ import division, print_function, absolute_import + +import tflearn + +# Growth Rate (12, 16, 32, ...) +k = 12 + +# Depth (40, 100, ...) +L = 40 +nb_layers = int((L - 4) / 3) + +# Data loading +from tflearn.datasets import cifar10 +(X, Y), (testX, testY) = cifar10.load_data() +Y = tflearn.data_utils.to_categorical(Y, 10) +testY = tflearn.data_utils.to_categorical(testY, 10) + +# Real-time data preprocessing +img_prep = tflearn.ImagePreprocessing() +img_prep.add_featurewise_zero_center(per_channel=True) + +# Real-time data augmentation +img_aug = tflearn.ImageAugmentation() +img_aug.add_random_flip_leftright() +img_aug.add_random_crop([32, 32], padding=4) + +# Building Residual Network +net = tflearn.input_data(shape=[None, 32, 32, 3], + data_preprocessing=img_prep, + data_augmentation=img_aug) +net = tflearn.conv_2d(net, 16, 3, regularizer='L2', weight_decay=0.0001) +net = tflearn.densenet_block(net, nb_layers, k) +net = tflearn.densenet_block(net, nb_layers, k) +net = tflearn.densenet_block(net, nb_layers, k) +net = tflearn.global_avg_pool(net) + +# Regression +net = tflearn.fully_connected(net, 10, activation='softmax') +opt = tflearn.SGD(0.1, lr_decay=0.1, decay_step=32000, staircase=True) +net = tflearn.regression(net, optimizer=opt, + loss='categorical_crossentropy') +# Training +model = tflearn.DNN(net, checkpoint_path='model_densenet_cifar10', + max_checkpoints=10, tensorboard_verbose=0, + clip_gradients=0.) + +model.fit(X, Y, n_epoch=200, validation_set=(testX, testY), + snapshot_epoch=False, snapshot_step=500, + show_metric=True, batch_size=128, shuffle=True, + run_id='densenet_cifar10') diff --git a/tflearn/__init__.py b/tflearn/__init__.py index 7ae8f3bd..54556932 100644 --- a/tflearn/__init__.py +++ b/tflearn/__init__.py @@ -46,7 +46,7 @@ highway_conv_2d, highway_conv_1d, max_pool_1d, avg_pool_1d, \ global_avg_pool, residual_block, residual_bottleneck, \ conv_2d_transpose, upsample_2d, conv_3d, max_pool_3d, avg_pool_3d, \ - resnext_block, upscore_layer, deconv_2d + resnext_block, upscore_layer, deconv_2d, densenet_block from .layers.core import input_data, dropout, custom_layer, reshape, \ flatten, activation, fully_connected, single_unit, highway, \ one_hot_encoding, time_distributed, multi_target_data diff --git a/tflearn/layers/__init__.py b/tflearn/layers/__init__.py index f3ecaedd..bfe72ae7 100644 --- a/tflearn/layers/__init__.py +++ b/tflearn/layers/__init__.py @@ -2,7 +2,7 @@ from .conv import conv_2d, max_pool_2d, avg_pool_2d, conv_1d, \ max_pool_1d, avg_pool_1d, residual_block, residual_bottleneck, \ highway_conv_1d, highway_conv_2d, upsample_2d, conv_3d, max_pool_3d, \ - avg_pool_3d, resnext_block, upscore_layer, deconv_2d + avg_pool_3d, resnext_block, upscore_layer, deconv_2d, densenet_block from .core import input_data, dropout, custom_layer, reshape, flatten, \ activation, fully_connected, single_unit, one_hot_encoding, time_distributed, \ multi_target_data diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index b1e2522f..896aa5a9 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -1694,6 +1694,133 @@ def resnext_block(incoming, nb_blocks, out_channels, cardinality, return resnext +def densenet_block(incoming, nb_layers, growth, bottleneck=True, + downsample=True, downsample_strides=2, activation='relu', + batch_norm=True, dropout=False, dropout_keep_prob=0.5, + weights_init='variance_scaling', regularizer='L2', + weight_decay=0.0001, bias=True, bias_init='zeros', + trainable=True, restore=True, reuse=False, scope=None, + name="DenseNetBlock"): + """ DenseNet Block. + + A DenseNet block as described in DenseNet paper. + + Input: + 4-D Tensor [batch, height, width, in_channels]. + + Output: + 4-D Tensor [batch, new height, new width, out_channels]. + + Arguments: + incoming: `Tensor`. Incoming 4-D Layer. + nb_blocks: `int`. Number of layer blocks. + growth: `int`. DenseNet 'growth': The number of convolutional + filters of each convolution. + bottleneck: `bool`. If True, add a 1x1 convolution before the 3x3 + convolution to reduce the number of input features map. + downsample: `bool`. If True, apply downsampling using + 'downsample_strides' for strides. + downsample_strides: `int`. The strides to use when downsampling. + activation: `str` (name) or `function` (returning a `Tensor`). + Activation applied to this layer (see tflearn.activations). + Default: 'linear'. + batch_norm: `bool`. If True, apply batch normalization. + dropout: `bool`. If True, apply dropout. Use 'dropout_keep_prob' to + specify the keep probability. + dropout_keep_prob: `float`. Keep probability parameter for dropout. + bias: `bool`. If True, a bias is used. + weights_init: `str` (name) or `Tensor`. Weights initialization. + (see tflearn.initializations) Default: 'uniform_scaling'. + bias_init: `str` (name) or `tf.Tensor`. Bias initialization. + (see tflearn.initializations) Default: 'zeros'. + regularizer: `str` (name) or `Tensor`. Add a regularizer to this + layer weights (see tflearn.regularizers). Default: None. + weight_decay: `float`. Regularizer decay parameter. Default: 0.001. + trainable: `bool`. If True, weights will be trainable. + restore: `bool`. If True, this layer weights will be restored when + loading a model. + reuse: `bool`. If True and 'scope' is provided, this layer variables + will be reused (shared). + scope: `str`. Define this layer scope (optional). A scope can be + used to share variables between layers. Note that scope will + override name. + name: A name for this layer (optional). Default: 'ResNeXtBlock'. + + References: + Densely Connected Convolutional Networks, G. Huang, Z. Liu, + K. Q. Weinberger, L. van der Maaten. 2016. + + Links: + [https://arxiv.org/abs/1608.06993] + (https://arxiv.org/abs/1608.06993) + + """ + densenet = incoming + + with tf.variable_scope(scope, default_name=name, values=[incoming], + reuse=reuse) as scope: + + for i in range(nb_layers): + + # Identity + conn = densenet + + # 1x1 Conv layer of the bottleneck block + if bottleneck: + if batch_norm: + densenet = tflearn.batch_normalization(densenet) + densenet = tflearn.activation(densenet, activation) + densenet = conv_2d(densenet, nb_filter=growth, + filter_size=1, + bias=bias, + weights_init=weights_init, + bias_init=bias_init, + regularizer=regularizer, + weight_decay=weight_decay, + trainable=trainable, + restore=restore) + + # 3x3 Conv layer + if batch_norm: + densenet = tflearn.batch_normalization(densenet) + densenet = tflearn.activation(densenet, activation) + densenet = conv_2d(densenet, nb_filter=growth, + filter_size=3, + bias=bias, + weights_init=weights_init, + bias_init=bias_init, + regularizer=regularizer, + weight_decay=weight_decay, + trainable=trainable, + restore=restore) + + # Connections + densenet = tf.concat([densenet, conn], 3) + + # 1x1 Transition Conv + if batch_norm: + densenet = tflearn.batch_normalization(densenet) + densenet = tflearn.activation(densenet, activation) + densenet = conv_2d(densenet, nb_filter=growth, + filter_size=1, + bias=bias, + weights_init=weights_init, + bias_init=bias_init, + regularizer=regularizer, + weight_decay=weight_decay, + trainable=trainable, + restore=restore) + if dropout: + densenet = tflearn.dropout(densenet, keep_prob=dropout_keep_prob) + + # Downsampling + if downsample: + densenet = tflearn.avg_pool_2d(densenet, kernel_size=2, + strides=downsample_strides) + + return densenet + + def highway_conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', activation='linear', weights_init='uniform_scaling', bias_init='zeros', regularizer=None, weight_decay=0.001, From 6d6c8a678932c01854e456924e21e273d7bd607d Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Wed, 26 Jul 2017 23:36:30 +0100 Subject: [PATCH 11/61] densenet fixes --- examples/images/densenet.py | 2 +- tflearn/__init__.py | 2 +- tflearn/layers/conv.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/images/densenet.py b/examples/images/densenet.py index afb09db9..82ad74dc 100644 --- a/examples/images/densenet.py +++ b/examples/images/densenet.py @@ -51,7 +51,7 @@ # Regression net = tflearn.fully_connected(net, 10, activation='softmax') -opt = tflearn.SGD(0.1, lr_decay=0.1, decay_step=32000, staircase=True) +opt = tflearn.Nesterov(0.1, lr_decay=0.1, decay_step=32000, staircase=True) net = tflearn.regression(net, optimizer=opt, loss='categorical_crossentropy') # Training diff --git a/tflearn/__init__.py b/tflearn/__init__.py index 54556932..80fd9635 100644 --- a/tflearn/__init__.py +++ b/tflearn/__init__.py @@ -31,7 +31,7 @@ # Direct ops inclusion from .optimizers import SGD, AdaGrad, Adam, RMSProp, Momentum, Ftrl, AdaDelta, \ - ProximalAdaGrad + ProximalAdaGrad, Nesterov from .activations import linear, tanh, sigmoid, softmax, softplus, softsign,\ relu, relu6, leaky_relu, prelu, elu, crelu, selu from .variables import variable, get_all_trainable_variable, \ diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index 896aa5a9..6fd50fb1 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -1757,10 +1757,10 @@ def densenet_block(incoming, nb_layers, growth, bottleneck=True, """ densenet = incoming - with tf.variable_scope(scope, default_name=name, values=[incoming], - reuse=reuse) as scope: + for i in range(nb_layers): - for i in range(nb_layers): + with tf.variable_scope(scope, default_name=name, values=[incoming], + reuse=reuse) as scope: # Identity conn = densenet From 6b05603010b9002e5ad8999fb5d54eabaa0a9fb6 Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Wed, 26 Jul 2017 23:51:53 +0100 Subject: [PATCH 12/61] fix bug --- tflearn/layers/conv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index 6fd50fb1..896aa5a9 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -1757,10 +1757,10 @@ def densenet_block(incoming, nb_layers, growth, bottleneck=True, """ densenet = incoming - for i in range(nb_layers): + with tf.variable_scope(scope, default_name=name, values=[incoming], + reuse=reuse) as scope: - with tf.variable_scope(scope, default_name=name, values=[incoming], - reuse=reuse) as scope: + for i in range(nb_layers): # Identity conn = densenet From d28030a59f965c0f433fe404599436eb9ed902d8 Mon Sep 17 00:00:00 2001 From: zhengyao jiang Date: Sat, 29 Jul 2017 20:53:13 +0800 Subject: [PATCH 13/61] fix elu activation function (#853) --- tflearn/activations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/activations.py b/tflearn/activations.py index 946fdb31..28f61382 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -303,4 +303,4 @@ def selu(x): """ alpha = 1.6732632423543772848170429916717 scale = 1.0507009873554804934193349852946 - return scale * tf.nn.elu(x, alpha) + return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x)) From 6669cc82d1ed4b59df80ff43a5da8c87fe2fcb90 Mon Sep 17 00:00:00 2001 From: windog18 Date: Sat, 5 Aug 2017 19:44:31 +0800 Subject: [PATCH 14/61] fixed error on summaries (#846) --- examples/extending_tensorflow/summaries.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/examples/extending_tensorflow/summaries.py b/examples/extending_tensorflow/summaries.py index 79e93e36..02a823d7 100644 --- a/examples/extending_tensorflow/summaries.py +++ b/examples/extending_tensorflow/summaries.py @@ -74,11 +74,13 @@ def dnn(x): return x net = dnn(X) - loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(net, Y)) - optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) - accuracy = tf.reduce_mean( - tf.cast(tf.equal(tf.argmax(net, 1), tf.argmax(Y, 1)), tf.float32), - name="acc") + + with tf.name_scope('Summaries'): + loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=net,labels=Y)) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) + accuracy = tf.reduce_mean( + tf.cast(tf.equal(tf.argmax(net, 1), tf.argmax(Y, 1)), tf.float32), + name="acc") # construct two varaibles to add as additional "valiation monitors" # these varaibles are evaluated each time validation happens (eg at a snapshot) @@ -92,11 +94,10 @@ def dnn(x): with tf.name_scope('CustomMonitor'): test_var = tf.reduce_sum(tf.cast(net, tf.float32), name="test_var") test_const = tf.constant(32.0, name="custom_constant") - - # Define a train op + # Define a train op trainop = tflearn.TrainOp(loss=loss, optimizer=optimizer, - validation_monitors=[test_var, test_const], - metric=accuracy, batch_size=128) + validation_monitors=[test_var, test_const], + metric=accuracy, batch_size=128) # Tensorboard logs stored in /tmp/tflearn_logs/. Using verbose level 2. trainer = tflearn.Trainer(train_ops=trainop, From 8f12d066d75d08d319901e9d5a2564a101943a8d Mon Sep 17 00:00:00 2001 From: Torrencem Date: Sun, 13 Aug 2017 15:35:47 -0400 Subject: [PATCH 15/61] Fix Grayscale Image Shape (#869) * Fix Grayscale Image Shape When using grayscale images, making the shape [None, width, height, 1] will allow you to use it directly with an input_layer, since it's a 4-D tensor, as apposed to [None, width, height]. Color images are 4-D by default ([None, width, height, 3]) * Fix Grayscale Image Shape Changed the addition to work for all versions of python (tuple unpacking like that is only 3.x) * Fix Typo Added a paren --- tflearn/data_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 65ad2118..8f81d4d6 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -832,6 +832,8 @@ def preload(self, path, image_shape, normalize=True, grayscale=False): if grayscale: img = convert_color(img, 'L') img = pil_to_nparray(img) + if grayscale: + img = np.reshape(img, img.shape + (1,)) if normalize: img /= 255. return img From bf9dbbcf602b018b78d20c6a1d41cf625314b3ee Mon Sep 17 00:00:00 2001 From: Amitayus Date: Mon, 21 Aug 2017 20:59:30 +0800 Subject: [PATCH 16/61] Coding format! (#877) The code are so ugly, so I format it. --- examples/images/googlenet.py | 86 +++++++++++++++++------------------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/examples/images/googlenet.py b/examples/images/googlenet.py index 6f510926..9f26c557 100644 --- a/examples/images/googlenet.py +++ b/examples/images/googlenet.py @@ -23,38 +23,37 @@ import tflearn.datasets.oxflower17 as oxflower17 X, Y = oxflower17.load_data(one_hot=True, resize_pics=(227, 227)) - network = input_data(shape=[None, 227, 227, 3]) -conv1_7_7 = conv_2d(network, 64, 7, strides=2, activation='relu', name = 'conv1_7_7_s2') -pool1_3_3 = max_pool_2d(conv1_7_7, 3,strides=2) +conv1_7_7 = conv_2d(network, 64, 7, strides=2, activation='relu', name='conv1_7_7_s2') +pool1_3_3 = max_pool_2d(conv1_7_7, 3, strides=2) pool1_3_3 = local_response_normalization(pool1_3_3) -conv2_3_3_reduce = conv_2d(pool1_3_3, 64,1, activation='relu',name = 'conv2_3_3_reduce') -conv2_3_3 = conv_2d(conv2_3_3_reduce, 192,3, activation='relu', name='conv2_3_3') +conv2_3_3_reduce = conv_2d(pool1_3_3, 64, 1, activation='relu', name='conv2_3_3_reduce') +conv2_3_3 = conv_2d(conv2_3_3_reduce, 192, 3, activation='relu', name='conv2_3_3') conv2_3_3 = local_response_normalization(conv2_3_3) pool2_3_3 = max_pool_2d(conv2_3_3, kernel_size=3, strides=2, name='pool2_3_3_s2') + +# 3a inception_3a_1_1 = conv_2d(pool2_3_3, 64, 1, activation='relu', name='inception_3a_1_1') -inception_3a_3_3_reduce = conv_2d(pool2_3_3, 96,1, activation='relu', name='inception_3a_3_3_reduce') -inception_3a_3_3 = conv_2d(inception_3a_3_3_reduce, 128,filter_size=3, activation='relu', name = 'inception_3a_3_3') -inception_3a_5_5_reduce = conv_2d(pool2_3_3,16, filter_size=1,activation='relu', name ='inception_3a_5_5_reduce' ) -inception_3a_5_5 = conv_2d(inception_3a_5_5_reduce, 32, filter_size=5, activation='relu', name= 'inception_3a_5_5') -inception_3a_pool = max_pool_2d(pool2_3_3, kernel_size=3, strides=1, ) +inception_3a_3_3_reduce = conv_2d(pool2_3_3, 96, 1, activation='relu', name='inception_3a_3_3_reduce') +inception_3a_3_3 = conv_2d(inception_3a_3_3_reduce, 128, filter_size=3, activation='relu', name='inception_3a_3_3') +inception_3a_5_5_reduce = conv_2d(pool2_3_3, 16, filter_size=1, activation='relu', name='inception_3a_5_5_reduce') +inception_3a_5_5 = conv_2d(inception_3a_5_5_reduce, 32, filter_size=5, activation='relu', name='inception_3a_5_5') +inception_3a_pool = max_pool_2d(pool2_3_3, kernel_size=3, strides=1, name='inception_3a_pool') inception_3a_pool_1_1 = conv_2d(inception_3a_pool, 32, filter_size=1, activation='relu', name='inception_3a_pool_1_1') - -# merge the inception_3a__ inception_3a_output = merge([inception_3a_1_1, inception_3a_3_3, inception_3a_5_5, inception_3a_pool_1_1], mode='concat', axis=3) -inception_3b_1_1 = conv_2d(inception_3a_output, 128,filter_size=1,activation='relu', name= 'inception_3b_1_1' ) +# 3b +inception_3b_1_1 = conv_2d(inception_3a_output, 128, filter_size=1, activation='relu', name='inception_3b_1_1') inception_3b_3_3_reduce = conv_2d(inception_3a_output, 128, filter_size=1, activation='relu', name='inception_3b_3_3_reduce') -inception_3b_3_3 = conv_2d(inception_3b_3_3_reduce, 192, filter_size=3, activation='relu',name='inception_3b_3_3') -inception_3b_5_5_reduce = conv_2d(inception_3a_output, 32, filter_size=1, activation='relu', name = 'inception_3b_5_5_reduce') -inception_3b_5_5 = conv_2d(inception_3b_5_5_reduce, 96, filter_size=5, name = 'inception_3b_5_5') +inception_3b_3_3 = conv_2d(inception_3b_3_3_reduce, 192, filter_size=3, activation='relu', name='inception_3b_3_3') +inception_3b_5_5_reduce = conv_2d(inception_3a_output, 32, filter_size=1, activation='relu', name='inception_3b_5_5_reduce') +inception_3b_5_5 = conv_2d(inception_3b_5_5_reduce, 96, filter_size=5, name='inception_3b_5_5') inception_3b_pool = max_pool_2d(inception_3a_output, kernel_size=3, strides=1, name='inception_3b_pool') -inception_3b_pool_1_1 = conv_2d(inception_3b_pool, 64, filter_size=1,activation='relu', name='inception_3b_pool_1_1') - -#merge the inception_3b_* -inception_3b_output = merge([inception_3b_1_1, inception_3b_3_3, inception_3b_5_5, inception_3b_pool_1_1], mode='concat',axis=3,name='inception_3b_output') - +inception_3b_pool_1_1 = conv_2d(inception_3b_pool, 64, filter_size=1, activation='relu', name='inception_3b_pool_1_1') +inception_3b_output = merge([inception_3b_1_1, inception_3b_3_3, inception_3b_5_5, inception_3b_pool_1_1], mode='concat', axis=3, name='inception_3b_output') pool3_3_3 = max_pool_2d(inception_3b_output, kernel_size=3, strides=2, name='pool3_3_3') + +# 4a inception_4a_1_1 = conv_2d(pool3_3_3, 192, filter_size=1, activation='relu', name='inception_4a_1_1') inception_4a_3_3_reduce = conv_2d(pool3_3_3, 96, filter_size=1, activation='relu', name='inception_4a_3_3_reduce') inception_4a_3_3 = conv_2d(inception_4a_3_3_reduce, 208, filter_size=3, activation='relu', name='inception_4a_3_3') @@ -62,33 +61,29 @@ inception_4a_5_5 = conv_2d(inception_4a_5_5_reduce, 48, filter_size=5, activation='relu', name='inception_4a_5_5') inception_4a_pool = max_pool_2d(pool3_3_3, kernel_size=3, strides=1, name='inception_4a_pool') inception_4a_pool_1_1 = conv_2d(inception_4a_pool, 64, filter_size=1, activation='relu', name='inception_4a_pool_1_1') - inception_4a_output = merge([inception_4a_1_1, inception_4a_3_3, inception_4a_5_5, inception_4a_pool_1_1], mode='concat', axis=3, name='inception_4a_output') - +# 4b inception_4b_1_1 = conv_2d(inception_4a_output, 160, filter_size=1, activation='relu', name='inception_4a_1_1') inception_4b_3_3_reduce = conv_2d(inception_4a_output, 112, filter_size=1, activation='relu', name='inception_4b_3_3_reduce') inception_4b_3_3 = conv_2d(inception_4b_3_3_reduce, 224, filter_size=3, activation='relu', name='inception_4b_3_3') inception_4b_5_5_reduce = conv_2d(inception_4a_output, 24, filter_size=1, activation='relu', name='inception_4b_5_5_reduce') inception_4b_5_5 = conv_2d(inception_4b_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4b_5_5') - inception_4b_pool = max_pool_2d(inception_4a_output, kernel_size=3, strides=1, name='inception_4b_pool') inception_4b_pool_1_1 = conv_2d(inception_4b_pool, 64, filter_size=1, activation='relu', name='inception_4b_pool_1_1') - inception_4b_output = merge([inception_4b_1_1, inception_4b_3_3, inception_4b_5_5, inception_4b_pool_1_1], mode='concat', axis=3, name='inception_4b_output') - -inception_4c_1_1 = conv_2d(inception_4b_output, 128, filter_size=1, activation='relu',name='inception_4c_1_1') +# 4c +inception_4c_1_1 = conv_2d(inception_4b_output, 128, filter_size=1, activation='relu', name='inception_4c_1_1') inception_4c_3_3_reduce = conv_2d(inception_4b_output, 128, filter_size=1, activation='relu', name='inception_4c_3_3_reduce') inception_4c_3_3 = conv_2d(inception_4c_3_3_reduce, 256, filter_size=3, activation='relu', name='inception_4c_3_3') inception_4c_5_5_reduce = conv_2d(inception_4b_output, 24, filter_size=1, activation='relu', name='inception_4c_5_5_reduce') inception_4c_5_5 = conv_2d(inception_4c_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4c_5_5') - inception_4c_pool = max_pool_2d(inception_4b_output, kernel_size=3, strides=1) inception_4c_pool_1_1 = conv_2d(inception_4c_pool, 64, filter_size=1, activation='relu', name='inception_4c_pool_1_1') +inception_4c_output = merge([inception_4c_1_1, inception_4c_3_3, inception_4c_5_5, inception_4c_pool_1_1], mode='concat', axis=3, name='inception_4c_output') -inception_4c_output = merge([inception_4c_1_1, inception_4c_3_3, inception_4c_5_5, inception_4c_pool_1_1], mode='concat', axis=3,name='inception_4c_output') - +# 4d inception_4d_1_1 = conv_2d(inception_4c_output, 112, filter_size=1, activation='relu', name='inception_4d_1_1') inception_4d_3_3_reduce = conv_2d(inception_4c_output, 144, filter_size=1, activation='relu', name='inception_4d_3_3_reduce') inception_4d_3_3 = conv_2d(inception_4d_3_3_reduce, 288, filter_size=3, activation='relu', name='inception_4d_3_3') @@ -96,9 +91,9 @@ inception_4d_5_5 = conv_2d(inception_4d_5_5_reduce, 64, filter_size=5, activation='relu', name='inception_4d_5_5') inception_4d_pool = max_pool_2d(inception_4c_output, kernel_size=3, strides=1, name='inception_4d_pool') inception_4d_pool_1_1 = conv_2d(inception_4d_pool, 64, filter_size=1, activation='relu', name='inception_4d_pool_1_1') - inception_4d_output = merge([inception_4d_1_1, inception_4d_3_3, inception_4d_5_5, inception_4d_pool_1_1], mode='concat', axis=3, name='inception_4d_output') +# 4e inception_4e_1_1 = conv_2d(inception_4d_output, 256, filter_size=1, activation='relu', name='inception_4e_1_1') inception_4e_3_3_reduce = conv_2d(inception_4d_output, 160, filter_size=1, activation='relu', name='inception_4e_3_3_reduce') inception_4e_3_3 = conv_2d(inception_4e_3_3_reduce, 320, filter_size=3, activation='relu', name='inception_4e_3_3') @@ -106,41 +101,42 @@ inception_4e_5_5 = conv_2d(inception_4e_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_4e_5_5') inception_4e_pool = max_pool_2d(inception_4d_output, kernel_size=3, strides=1, name='inception_4e_pool') inception_4e_pool_1_1 = conv_2d(inception_4e_pool, 128, filter_size=1, activation='relu', name='inception_4e_pool_1_1') - - -inception_4e_output = merge([inception_4e_1_1, inception_4e_3_3, inception_4e_5_5,inception_4e_pool_1_1],axis=3, mode='concat') - +inception_4e_output = merge([inception_4e_1_1, inception_4e_3_3, inception_4e_5_5, inception_4e_pool_1_1], axis=3, mode='concat') pool4_3_3 = max_pool_2d(inception_4e_output, kernel_size=3, strides=2, name='pool_3_3') - +# 5a inception_5a_1_1 = conv_2d(pool4_3_3, 256, filter_size=1, activation='relu', name='inception_5a_1_1') inception_5a_3_3_reduce = conv_2d(pool4_3_3, 160, filter_size=1, activation='relu', name='inception_5a_3_3_reduce') inception_5a_3_3 = conv_2d(inception_5a_3_3_reduce, 320, filter_size=3, activation='relu', name='inception_5a_3_3') inception_5a_5_5_reduce = conv_2d(pool4_3_3, 32, filter_size=1, activation='relu', name='inception_5a_5_5_reduce') inception_5a_5_5 = conv_2d(inception_5a_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_5a_5_5') inception_5a_pool = max_pool_2d(pool4_3_3, kernel_size=3, strides=1, name='inception_5a_pool') -inception_5a_pool_1_1 = conv_2d(inception_5a_pool, 128, filter_size=1,activation='relu', name='inception_5a_pool_1_1') - -inception_5a_output = merge([inception_5a_1_1, inception_5a_3_3, inception_5a_5_5, inception_5a_pool_1_1], axis=3,mode='concat') +inception_5a_pool_1_1 = conv_2d(inception_5a_pool, 128, filter_size=1, activation='relu', name='inception_5a_pool_1_1') +inception_5a_output = merge([inception_5a_1_1, inception_5a_3_3, inception_5a_5_5, inception_5a_pool_1_1], axis=3, mode='concat') - -inception_5b_1_1 = conv_2d(inception_5a_output, 384, filter_size=1,activation='relu', name='inception_5b_1_1') +# 5b +inception_5b_1_1 = conv_2d(inception_5a_output, 384, filter_size=1, activation='relu', name='inception_5b_1_1') inception_5b_3_3_reduce = conv_2d(inception_5a_output, 192, filter_size=1, activation='relu', name='inception_5b_3_3_reduce') -inception_5b_3_3 = conv_2d(inception_5b_3_3_reduce, 384, filter_size=3,activation='relu', name='inception_5b_3_3') +inception_5b_3_3 = conv_2d(inception_5b_3_3_reduce, 384, filter_size=3, activation='relu', name='inception_5b_3_3') inception_5b_5_5_reduce = conv_2d(inception_5a_output, 48, filter_size=1, activation='relu', name='inception_5b_5_5_reduce') -inception_5b_5_5 = conv_2d(inception_5b_5_5_reduce,128, filter_size=5, activation='relu', name='inception_5b_5_5' ) +inception_5b_5_5 = conv_2d(inception_5b_5_5_reduce, 128, filter_size=5, activation='relu', name='inception_5b_5_5') inception_5b_pool = max_pool_2d(inception_5a_output, kernel_size=3, strides=1, name='inception_5b_pool') inception_5b_pool_1_1 = conv_2d(inception_5b_pool, 128, filter_size=1, activation='relu', name='inception_5b_pool_1_1') inception_5b_output = merge([inception_5b_1_1, inception_5b_3_3, inception_5b_5_5, inception_5b_pool_1_1], axis=3, mode='concat') - pool5_7_7 = avg_pool_2d(inception_5b_output, kernel_size=7, strides=1) pool5_7_7 = dropout(pool5_7_7, 0.4) -loss = fully_connected(pool5_7_7, 17,activation='softmax') + +# fc +loss = fully_connected(pool5_7_7, 17, activation='softmax') network = regression(loss, optimizer='momentum', loss='categorical_crossentropy', learning_rate=0.001) + +# to train model = tflearn.DNN(network, checkpoint_path='model_googlenet', max_checkpoints=1, tensorboard_verbose=2) + model.fit(X, Y, n_epoch=1000, validation_set=0.1, shuffle=True, show_metric=True, batch_size=64, snapshot_step=200, snapshot_epoch=False, run_id='googlenet_oxflowers17') + From b69546ba6dfe1a9c01299297a3f0ae413a91de04 Mon Sep 17 00:00:00 2001 From: Legor Date: Fri, 1 Sep 2017 22:54:44 +0200 Subject: [PATCH 17/61] When using the image_preloader the images can be loaded from an URL. (#875) --- tflearn/data_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 8f81d4d6..d3f6b696 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -8,6 +8,9 @@ import pickle import csv import warnings +from urllib.parse import urlparse +from io import BytesIO +from urllib import request """ Preprocessing provides some useful functions to preprocess data before @@ -538,7 +541,15 @@ class containing the images to classify. def load_image(in_image): """ Load an image, returns PIL.Image. """ - img = Image.open(in_image) + # if the path appears to be an URL + if urlparse(in_image).scheme in ('http', 'https',): + # set up the byte stream + img_stream = BytesIO(request.urlopen(in_image).read()) + # and read in as PIL image + img = Image.open(img_stream) + else: + # else use it as local file path + img = Image.open(in_image) return img From cca486ca8fc6f32b0f1eefc7ced9cc89aaa3b381 Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Fri, 1 Sep 2017 21:58:22 +0100 Subject: [PATCH 18/61] fix py2 comp --- tflearn/data_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index d3f6b696..8ae6683a 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -8,9 +8,14 @@ import pickle import csv import warnings -from urllib.parse import urlparse +try: #py3 + from urllib.parse import urlparse + from urllib import request +except: #py2 + from urlparse import urlparse + from six.moves.urllib import request from io import BytesIO -from urllib import request + """ Preprocessing provides some useful functions to preprocess data before From b6d2eb49303685dd440bf5b9ba326775b0880828 Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Fri, 1 Sep 2017 22:15:07 +0100 Subject: [PATCH 19/61] fix #892 (numpy weights compatibility) --- tflearn/layers/conv.py | 51 ++++++++++++++++++++++++++++++++++++------ tflearn/layers/core.py | 12 ++++++++-- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index 896aa5a9..afda9142 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -77,6 +77,8 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -89,9 +91,12 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -195,6 +200,8 @@ def conv_2d_transpose(incoming, nb_filter, filter_size, output_shape, W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -206,9 +213,12 @@ def conv_2d_transpose(incoming, nb_filter, filter_size, output_shape, b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -344,6 +354,8 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -356,9 +368,12 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -474,6 +489,8 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -486,9 +503,12 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -817,6 +837,8 @@ def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -828,9 +850,12 @@ def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -1022,6 +1047,8 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -1033,9 +1060,12 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -1139,6 +1169,8 @@ def conv_3d_transpose(incoming, nb_filter, filter_size, output_shape, W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) @@ -1150,9 +1182,12 @@ def conv_3d_transpose(incoming, nb_filter, filter_size, output_shape, b = None if bias: + b_shape = [nb_filter] if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = vs.variable('b', shape=nb_filter, initializer=bias_init, + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None + b = vs.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) @@ -2013,6 +2048,8 @@ def highway_conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', W_init = weights_init if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) diff --git a/tflearn/layers/core.py b/tflearn/layers/core.py index 46d8df83..f38cac6d 100644 --- a/tflearn/layers/core.py +++ b/tflearn/layers/core.py @@ -147,21 +147,29 @@ def fully_connected(incoming, n_units, activation='linear', bias=True, name = scope.name W_init = weights_init + filter_size = [n_inputs, n_units] if isinstance(weights_init, str): W_init = initializations.get(weights_init)() + elif type(W_init) in [tf.Tensor, np.ndarray, list]: + filter_size = None W_regul = None if regularizer is not None: W_regul = lambda x: losses.get(regularizer)(x, weight_decay) - W = va.variable('W', shape=[n_inputs, n_units], regularizer=W_regul, + W = va.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W) b = None if bias: + b_shape = [n_units] + if isinstance(bias_init, str): + bias_init = initializations.get(bias_init)() + elif type(bias_init) in [tf.Tensor, np.ndarray, list]: + b_shape = None if isinstance(bias_init, str): bias_init = initializations.get(bias_init)() - b = va.variable('b', shape=[n_units], initializer=bias_init, + b = va.variable('b', shape=b_shape, initializer=bias_init, trainable=trainable, restore=restore) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, b) From 3e0c3298ff508394f3ef191bcd7d732eb8860b2e Mon Sep 17 00:00:00 2001 From: Chris Oelmueller Date: Tue, 19 Sep 2017 02:08:23 +0200 Subject: [PATCH 20/61] [Docs] Convolution layers: Typo fixes (#903) * [Docs] Convolution layers: Typo fixes The markdown output should be less confused now. * [Docs] Ftrl Proximal optimizer: Typo fix Another missing `backtick`. --- tflearn/layers/conv.py | 8 ++++---- tflearn/optimizers.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index afda9142..8d7a3a7a 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -31,7 +31,7 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', incoming: `Tensor`. Incoming 4-D Tensor. nb_filter: `int`. The number of convolutional filters. filter_size: `int` or `list of int`. Size of filters. - strides: 'int` or list of `int`. Strides of conv operation. + strides: `int` or list of `int`. Strides of conv operation. Default: [1 1 1 1]. padding: `str` from `"same", "valid"`. Padding algo to use. Default: 'same'. @@ -306,7 +306,7 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', incoming: `Tensor`. Incoming 4-D Tensor. nb_filter: `int`. The number of convolutional filters. filter_size: `int` or `list of int`. Size of filters. - rate: 'int`. A positive int32. The stride with which we sample input + rate: `int`. A positive int32. The stride with which we sample input values across the height and width dimensions. Equivalently, the rate by which we upsample the filter values by inserting zeros across the height and width dimensions. In the literature, the @@ -440,7 +440,7 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, incoming: `Tensor`. Incoming 4-D Tensor. channel_multiplier: `int`. The number of channels to expand to. filter_size: `int` or `list of int`. Size of filters. - strides: 'int` or list of `int`. Strides of conv operation. + strides: `int` or list of `int`. Strides of conv operation. Default: [1 1 1 1]. padding: `str` from `"same", "valid"`. Padding algo to use. Default: 'same'. @@ -1001,7 +1001,7 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', incoming: `Tensor`. Incoming 5-D Tensor. nb_filter: `int`. The number of convolutional filters. filter_size: `int` or `list of int`. Size of filters. - strides: 'int` or list of `int`. Strides of conv operation. + strides: `int` or list of `int`. Strides of conv operation. Default: [1 1 1 1 1]. Must have strides[0] = strides[4] = 1. padding: `str` from `"same", "valid"`. Padding algo to use. Default: 'same'. diff --git a/tflearn/optimizers.py b/tflearn/optimizers.py index 3f804572..aab0d9c4 100644 --- a/tflearn/optimizers.py +++ b/tflearn/optimizers.py @@ -384,7 +384,7 @@ class Ftrl(Optimizer): Only positive values are allowed. l1_regularization_strength: `float`. Must be less or equal to zero. l2_regularization_strength: `float`. Must be less or equal to zero. - use_locking: bool`. If True use locks for update operation. + use_locking: `bool`. If True use locks for update operation. name: `str`. Optional name prefix for the operations created when applying gradients. Defaults to "Ftrl". From 16a3bbca5b8f5879ad54849f7c8fec23609863f4 Mon Sep 17 00:00:00 2001 From: Ilari Pihlajisto Date: Wed, 20 Sep 2017 23:41:39 +0300 Subject: [PATCH 21/61] convert RGBA to RGB in build_hdf5_image_dataset (#904) --- tflearn/data_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 8ae6683a..e938626e 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -419,7 +419,7 @@ class containing the images to classify. img = resize_image(img, image_shape[0], image_shape[1]) if grayscale: img = convert_color(img, 'L') - elif img.mode == 'L': + elif img.mode == 'L' or img.mode == 'RGBA': img = convert_color(img, 'RGB') img = pil_to_nparray(img) From 06067ccc8053ffec4421ce635ce06f03ea6d4ac5 Mon Sep 17 00:00:00 2001 From: Christopher Brust Date: Tue, 26 Sep 2017 19:46:32 -0400 Subject: [PATCH 22/61] Fixes the seq2seq example (#916) In the example, we point to the legacy seq2seq class in Tensorflow. In summaries.py, the format of the tag has changed in Tensorflow; this breaks the seq2seq example, so we hotfix the particular tag that's a problem in the seq2seq example. --- examples/nlp/seq2seq_example.py | 2 +- tflearn/summaries.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/nlp/seq2seq_example.py b/examples/nlp/seq2seq_example.py index bb860eb6..f8a4ac3c 100644 --- a/examples/nlp/seq2seq_example.py +++ b/examples/nlp/seq2seq_example.py @@ -14,7 +14,7 @@ import numpy as np import tensorflow as tf -from tensorflow.python.ops import seq2seq +from tensorflow.contrib.legacy_seq2seq.python.ops import seq2seq from tensorflow.python.ops import rnn_cell #----------------------------------------------------------------------------- diff --git a/tflearn/summaries.py b/tflearn/summaries.py index 425bbaa0..dbbe067d 100644 --- a/tflearn/summaries.py +++ b/tflearn/summaries.py @@ -179,6 +179,11 @@ def get_value_from_summary_string(tag, summary_str): `Exception` if tag not found. """ + + # Compatibility hotfix for the seq2seq example + if tag == u'acc:0/': + tag = u'acc_0/' + # Fix for TF 0.12 if tag[-1] == '/': tag = tag[:-1] From 82efb41887067516a52ef4f34f9a583699584757 Mon Sep 17 00:00:00 2001 From: Tianxiang Gao Date: Tue, 10 Oct 2017 00:55:37 +0800 Subject: [PATCH 23/61] remove the redundant parentheses (#925) --- tflearn/datasets/oxflower17.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/datasets/oxflower17.py b/tflearn/datasets/oxflower17.py index ef06e949..8e0b216d 100644 --- a/tflearn/datasets/oxflower17.py +++ b/tflearn/datasets/oxflower17.py @@ -46,7 +46,7 @@ def maybe_download(filename, source_url, work_directory): filepath, _ = urllib.request.urlretrieve(source_url + filename, filepath, reporthook) statinfo = os.stat(filepath) - print(('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')) + print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.') untar(filepath, work_directory) build_class_directories(os.path.join(work_directory, 'jpg')) From 8c1feff3df7ad75462d4ea1b881e75797b56efde Mon Sep 17 00:00:00 2001 From: John B Nelson Date: Tue, 17 Oct 2017 11:27:44 -0700 Subject: [PATCH 24/61] s/indexies/indices/ (#937) --- tflearn/data_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index e938626e..3f4e4715 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -232,7 +232,7 @@ def fit(self, raw_documents, unused_y=None): def fit_transform(self, raw_documents, unused_y=None): """ fit_transform. - Learn the vocabulary dictionary and return indexies of words. + Learn the vocabulary dictionary and return indices of words. Arguments: raw_documents: An iterable which yield either str or unicode. From 2f5853c2ec57908e42884e754ee66d0679f9fb83 Mon Sep 17 00:00:00 2001 From: Heros da Silva Araujo Date: Thu, 19 Oct 2017 13:44:46 -0200 Subject: [PATCH 25/61] Improve to_categorical function (#923) * Improve to_categorical function nb_classes not required anymore * Refactoring examples --- examples/basics/use_dask.py | 4 ++-- examples/basics/use_hdf5.py | 4 ++-- examples/images/convnet_cifar10.py | 4 ++-- examples/images/dcgan.py | 6 +++--- examples/images/densenet.py | 4 ++-- examples/images/network_in_network.py | 4 ++-- examples/images/residual_network_cifar10.py | 4 ++-- examples/images/resnext_cifar10.py | 4 ++-- examples/nlp/bidirectional_lstm.py | 4 ++-- examples/nlp/cnn_sentence_classification.py | 4 ++-- examples/nlp/dynamic_lstm.py | 4 ++-- examples/nlp/lstm.py | 4 ++-- examples/notebooks/spiral.ipynb | 4 ++-- tflearn/data_utils.py | 17 ++--------------- 14 files changed, 29 insertions(+), 42 deletions(-) diff --git a/examples/basics/use_dask.py b/examples/basics/use_dask.py index 9a082e4e..ead7f468 100644 --- a/examples/basics/use_dask.py +++ b/examples/basics/use_dask.py @@ -19,8 +19,8 @@ # Load CIFAR-10 Dataset from tflearn.datasets import cifar10 (X, Y), (X_test, Y_test) = cifar10.load_data() -Y = to_categorical(Y, 10) -Y_test = to_categorical(Y_test, 10) +Y = to_categorical(Y) +Y_test = to_categorical(Y_test) # Create DASK array using numpy arrays # (Note that it can work with HDF5 Dataset too) diff --git a/examples/basics/use_hdf5.py b/examples/basics/use_hdf5.py index 94fbe294..717c94b6 100644 --- a/examples/basics/use_hdf5.py +++ b/examples/basics/use_hdf5.py @@ -19,8 +19,8 @@ # CIFAR-10 Dataset from tflearn.datasets import cifar10 (X, Y), (X_test, Y_test) = cifar10.load_data() -Y = to_categorical(Y, 10) -Y_test = to_categorical(Y_test, 10) +Y = to_categorical(Y) +Y_test = to_categorical(Y_test) # Create a hdf5 dataset from CIFAR-10 numpy array import h5py diff --git a/examples/images/convnet_cifar10.py b/examples/images/convnet_cifar10.py index 7708d021..f044949f 100644 --- a/examples/images/convnet_cifar10.py +++ b/examples/images/convnet_cifar10.py @@ -23,8 +23,8 @@ from tflearn.datasets import cifar10 (X, Y), (X_test, Y_test) = cifar10.load_data() X, Y = shuffle(X, Y) -Y = to_categorical(Y, 10) -Y_test = to_categorical(Y_test, 10) +Y = to_categorical(Y) +Y_test = to_categorical(Y_test) # Real-time data preprocessing img_prep = ImagePreprocessing() diff --git a/examples/images/dcgan.py b/examples/images/dcgan.py index 4cfae763..cddaa519 100644 --- a/examples/images/dcgan.py +++ b/examples/images/dcgan.py @@ -99,15 +99,15 @@ def discriminator(x, reuse=False): # Prepare target data to feed to the discriminator (0: fake image, 1: real image) y_disc_fake = np.zeros(shape=[total_samples]) y_disc_real = np.ones(shape=[total_samples]) -y_disc_fake = tflearn.data_utils.to_categorical(y_disc_fake, 2) -y_disc_real = tflearn.data_utils.to_categorical(y_disc_real, 2) +y_disc_fake = tflearn.data_utils.to_categorical(y_disc_fake) +y_disc_real = tflearn.data_utils.to_categorical(y_disc_real) # Prepare input data to feed to the stacked generator/discriminator gen_noise = np.random.uniform(-1., 1., size=[total_samples, z_dim]) # Prepare target data to feed to the discriminator # Generator tries to fool the discriminator, thus target is 1 (e.g. real images) y_gen = np.ones(shape=[total_samples]) -y_gen = tflearn.data_utils.to_categorical(y_gen, 2) +y_gen = tflearn.data_utils.to_categorical(y_gen) # Start training, feed both noise and real images. gan.fit(X_inputs={'input_gen_noise': gen_noise, diff --git a/examples/images/densenet.py b/examples/images/densenet.py index 82ad74dc..106d278d 100644 --- a/examples/images/densenet.py +++ b/examples/images/densenet.py @@ -27,8 +27,8 @@ # Data loading from tflearn.datasets import cifar10 (X, Y), (testX, testY) = cifar10.load_data() -Y = tflearn.data_utils.to_categorical(Y, 10) -testY = tflearn.data_utils.to_categorical(testY, 10) +Y = tflearn.data_utils.to_categorical(Y) +testY = tflearn.data_utils.to_categorical(testY) # Real-time data preprocessing img_prep = tflearn.ImagePreprocessing() diff --git a/examples/images/network_in_network.py b/examples/images/network_in_network.py index 02b583a8..86335c3d 100644 --- a/examples/images/network_in_network.py +++ b/examples/images/network_in_network.py @@ -24,8 +24,8 @@ from tflearn.datasets import cifar10 (X, Y), (X_test, Y_test) = cifar10.load_data() X, Y = shuffle(X, Y) -Y = to_categorical(Y, 10) -Y_test = to_categorical(Y_test, 10) +Y = to_categorical(Y) +Y_test = to_categorical(Y_test) # Building 'Network In Network' network = input_data(shape=[None, 32, 32, 3]) diff --git a/examples/images/residual_network_cifar10.py b/examples/images/residual_network_cifar10.py index bd07ead4..694f98d7 100644 --- a/examples/images/residual_network_cifar10.py +++ b/examples/images/residual_network_cifar10.py @@ -26,8 +26,8 @@ # Data loading from tflearn.datasets import cifar10 (X, Y), (testX, testY) = cifar10.load_data() -Y = tflearn.data_utils.to_categorical(Y, 10) -testY = tflearn.data_utils.to_categorical(testY, 10) +Y = tflearn.data_utils.to_categorical(Y) +testY = tflearn.data_utils.to_categorical(testY) # Real-time data preprocessing img_prep = tflearn.ImagePreprocessing() diff --git a/examples/images/resnext_cifar10.py b/examples/images/resnext_cifar10.py index 20077ead..7529f29b 100644 --- a/examples/images/resnext_cifar10.py +++ b/examples/images/resnext_cifar10.py @@ -24,8 +24,8 @@ # Data loading from tflearn.datasets import cifar10 (X, Y), (testX, testY) = cifar10.load_data() -Y = tflearn.data_utils.to_categorical(Y, 10) -testY = tflearn.data_utils.to_categorical(testY, 10) +Y = tflearn.data_utils.to_categorical(Y) +testY = tflearn.data_utils.to_categorical(testY) # Real-time data preprocessing img_prep = tflearn.ImagePreprocessing() diff --git a/examples/nlp/bidirectional_lstm.py b/examples/nlp/bidirectional_lstm.py index 2eafca9b..e9e90683 100644 --- a/examples/nlp/bidirectional_lstm.py +++ b/examples/nlp/bidirectional_lstm.py @@ -38,8 +38,8 @@ trainX = pad_sequences(trainX, maxlen=200, value=0.) testX = pad_sequences(testX, maxlen=200, value=0.) # Converting labels to binary vectors -trainY = to_categorical(trainY, nb_classes=2) -testY = to_categorical(testY, nb_classes=2) +trainY = to_categorical(trainY) +testY = to_categorical(testY) # Network building net = input_data(shape=[None, 200]) diff --git a/examples/nlp/cnn_sentence_classification.py b/examples/nlp/cnn_sentence_classification.py index f753ff0a..9f11f055 100644 --- a/examples/nlp/cnn_sentence_classification.py +++ b/examples/nlp/cnn_sentence_classification.py @@ -38,8 +38,8 @@ trainX = pad_sequences(trainX, maxlen=100, value=0.) testX = pad_sequences(testX, maxlen=100, value=0.) # Converting labels to binary vectors -trainY = to_categorical(trainY, nb_classes=2) -testY = to_categorical(testY, nb_classes=2) +trainY = to_categorical(trainY) +testY = to_categorical(testY) # Building convolutional network network = input_data(shape=[None, 100], name='input') diff --git a/examples/nlp/dynamic_lstm.py b/examples/nlp/dynamic_lstm.py index 7c18188a..aaf64457 100644 --- a/examples/nlp/dynamic_lstm.py +++ b/examples/nlp/dynamic_lstm.py @@ -37,8 +37,8 @@ trainX = pad_sequences(trainX, maxlen=100, value=0.) testX = pad_sequences(testX, maxlen=100, value=0.) # Converting labels to binary vectors -trainY = to_categorical(trainY, nb_classes=2) -testY = to_categorical(testY, nb_classes=2) +trainY = to_categorical(trainY) +testY = to_categorical(testY) # Network building net = tflearn.input_data([None, 100]) diff --git a/examples/nlp/lstm.py b/examples/nlp/lstm.py index 6a906245..9e69b06e 100644 --- a/examples/nlp/lstm.py +++ b/examples/nlp/lstm.py @@ -33,8 +33,8 @@ trainX = pad_sequences(trainX, maxlen=100, value=0.) testX = pad_sequences(testX, maxlen=100, value=0.) # Converting labels to binary vectors -trainY = to_categorical(trainY, nb_classes=2) -testY = to_categorical(testY, nb_classes=2) +trainY = to_categorical(trainY) +testY = to_categorical(testY) # Network building net = tflearn.input_data([None, 100]) diff --git a/examples/notebooks/spiral.ipynb b/examples/notebooks/spiral.ipynb index 220f63bf..28e12564 100644 --- a/examples/notebooks/spiral.ipynb +++ b/examples/notebooks/spiral.ipynb @@ -116,7 +116,7 @@ " gd = tf.train.GradientDescentOptimizer(learning_rate=1.0)\n", " net = tflearn.regression(net, optimizer=gd, loss='categorical_crossentropy')\n", "\n", - " Y = to_categorical(y, 3)\n", + " Y = to_categorical(y)\n", " lm = tflearn.DNN(net)\n", " lm.fit(X, Y, show_metric=True, batch_size=len(X), n_epoch=1000, snapshot_epoch=False)" ] @@ -195,7 +195,7 @@ " sgd = tflearn.SGD(learning_rate=1.0, lr_decay=0.96, decay_step=500)\n", " net = tflearn.regression(net, optimizer=sgd, loss='categorical_crossentropy')\n", "\n", - " Y = to_categorical(y, 3)\n", + " Y = to_categorical(y)\n", " model = tflearn.DNN(net)\n", " model.fit(X, Y, show_metric=True, batch_size=len(X), n_epoch=5000, snapshot_epoch=False)" ] diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 3f4e4715..33883dc5 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -33,7 +33,7 @@ # ======================= -def to_categorical(y, nb_classes): +def to_categorical(y): """ to_categorical. Convert class vector (integers from 0 to nb_classes) @@ -41,21 +41,8 @@ def to_categorical(y, nb_classes): Arguments: y: `array`. Class vector to convert. - nb_classes: `int`. Total number of classes. - """ - y = np.asarray(y, dtype='int32') - # high dimensional array warning - if len(y.shape) > 2: - warnings.warn('{}-dimensional array is used as input array.'.format(len(y.shape)), stacklevel=2) - # flatten high dimensional array - if len(y.shape) > 1: - y = y.reshape(-1) - if not nb_classes: - nb_classes = np.max(y)+1 - Y = np.zeros((len(y), nb_classes)) - Y[np.arange(len(y)),y] = 1. - return Y + return (y[:, None] == np.unique(y)).astype(np.float32) # ===================== From f6a947fe0f26bfe2fa18683d24a0040ec14c7b7f Mon Sep 17 00:00:00 2001 From: Aymeric Damien Date: Thu, 19 Oct 2017 20:07:48 +0100 Subject: [PATCH 26/61] update to_categorical temporary add the older 'nb_classes' arg for older code compatibility. --- tflearn/data_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 33883dc5..89b726e1 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -33,7 +33,7 @@ # ======================= -def to_categorical(y): +def to_categorical(y, nb_classes=None): """ to_categorical. Convert class vector (integers from 0 to nb_classes) @@ -41,6 +41,7 @@ def to_categorical(y): Arguments: y: `array`. Class vector to convert. + nb_classes: `unused`. Used for older code compatibility. """ return (y[:, None] == np.unique(y)).astype(np.float32) From 72bb85a3f638ae8c5f2604a8b266223515941eac Mon Sep 17 00:00:00 2001 From: Kyle Kosic Date: Wed, 15 Nov 2017 11:57:58 -0500 Subject: [PATCH 27/61] rnn return_seq as 3d-tensor (#953) --- tflearn/layers/recurrent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tflearn/layers/recurrent.py b/tflearn/layers/recurrent.py index 99d67d8f..f033a8ba 100644 --- a/tflearn/layers/recurrent.py +++ b/tflearn/layers/recurrent.py @@ -80,12 +80,12 @@ def _rnn_template(incoming, cell, dropout=None, return_seq=False, if dynamic: if return_seq: - o = outputs + o = tf.stack(outputs, 1) else: outputs = tf.transpose(tf.stack(outputs), [1, 0, 2]) o = advanced_indexing_op(outputs, sequence_length) else: - o = outputs if return_seq else outputs[-1] + o = tf.stack(outputs, 1) if return_seq else outputs[-1] # Track output tensor. tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, o) @@ -385,12 +385,12 @@ def bidirectional_rnn(incoming, rnncell_fw, rnncell_bw, return_seq=False, if dynamic: if return_seq: - o = outputs + o = tf.stack(outputs, 1) else: outputs = tf.transpose(tf.stack(outputs), [1, 0, 2]) o = advanced_indexing_op(outputs, sequence_length) else: - o = outputs if return_seq else outputs[-1] + o = tf.stack(outputs, 1) if return_seq else outputs[-1] sfw = states_fw sbw = states_bw From 0fb2a2d701c00b6445e9ea67cab929913ca1ffe3 Mon Sep 17 00:00:00 2001 From: plooney Date: Tue, 12 Dec 2017 07:54:50 +0000 Subject: [PATCH 28/61] Adding an upscore_layer for 3D. (#973) * Adding upscore3D layer * Changing upscore2d to be consistent with upscore3d. --- tflearn/layers/conv.py | 116 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 3 deletions(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index 8d7a3a7a..6f37c70f 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -681,7 +681,7 @@ def upscore_layer(incoming, num_classes, shape=None, kernel_size=4, 4-D Tensor [batch, height, width, in_channels]. Output: - 4-D Tensor [batch, pooled height, pooled width, in_channels]. + 4-D Tensor [pooled height, pooled width]. Arguments: incoming: `Tensor`. Incoming 4-D Layer to upsample. @@ -722,15 +722,15 @@ def upscore_layer(incoming, num_classes, shape=None, kernel_size=4, reuse=reuse) as scope: name = scope.name + in_shape = tf.shape(incoming) if shape is None: # Compute shape out of Bottom - in_shape = tf.shape(incoming) h = ((in_shape[1] - 1) * strides[1]) + 1 w = ((in_shape[2] - 1) * strides[1]) + 1 new_shape = [in_shape[0], h, w, num_classes] else: - new_shape = [shape[0], shape[1], shape[2], num_classes] + new_shape = [in_shape[0], shape[0], shape[1], num_classes] output_shape = tf.stack(new_shape) def get_deconv_filter(f_shape): @@ -769,6 +769,116 @@ def get_deconv_filter(f_shape): return deconv +def upscore_layer3d(incoming, num_classes, shape=None, kernel_size=4, + strides=2, trainable=True, restore=True, + reuse=False, scope=None, name='Upscore'): + """ Upscore. + + This implements the upscore layer as used in + (Fully Convolutional Networks)[http://arxiv.org/abs/1411.4038]. + The upscore layer is initialized as bilinear upsampling filter. + + Input: + 5-D Tensor [batch, height, width, depth, in_channels]. + + Output: + 5-D Tensor [batch, pooled height, pooled width, pooled depth, in_channels]. + + Arguments: + incoming: `Tensor`. Incoming 4-D Layer to upsample. + num_classes: `int`. Number of output feature maps. + shape: `list of int`. Dimension of the output map + [new height, new width, new depth]. For convinience four values + are allows [new height, new width, new depth, X], where X + is ignored. + kernel_size: 'int` or `list of int`. Upsampling kernel size. + strides: 'int` or `list of int`. Strides of conv operation. + Default: [1 2 2 2 1]. + trainable: `bool`. If True, weights will be trainable. + restore: `bool`. If True, this layer weights will be restored when + loading a model. + reuse: `bool`. If True and 'scope' is provided, this layer variables + will be reused (shared). + scope: `str`. Define this layer scope (optional). A scope can be + used to share variables between layers. Note that scope will + override name. + name: A name for this layer (optional). Default: 'Upscore'. + + Attributes: + scope: `Scope`. This layer scope. + + Links: + (Fully Convolutional Networks)[http://arxiv.org/abs/1411.4038] + + """ + input_shape = utils.get_incoming_shape(incoming) + assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D" + + strides = utils.autoformat_kernel_3d(strides) + filter_size = utils.autoformat_filter_conv3d(kernel_size, + num_classes, + input_shape[-1]) + + # Variable Scope fix for older TF + try: + vscope = tf.variable_scope(scope, default_name=name, values=[incoming], + reuse=reuse) + except Exception: + vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse) + + with vscope as scope: + name = scope.name + + in_shape = tf.shape(incoming) + if shape is None: + # Compute shape out of Bottom + + h = ((in_shape[1] - 1) * strides[1]) + 1 + w = ((in_shape[2] - 1) * strides[1]) + 1 + d = ((in_shape[3] - 1) * strides[1]) + 1 + new_shape = [in_shape[0], h, w, d, num_classes] + else: + new_shape = [in_shape[0], shape[0], shape[1], shape[2], num_classes] + output_shape = tf.stack(new_shape) + + def get_deconv_filter(f_shape): + """ + Create filter weights initialized as bilinear upsampling. + """ + width = f_shape[0] + heigh = f_shape[0] + depth = f_shape[0] + f = ceil(width/2.0) + c = (2 * f - 1 - f % 2) / (2.0 * f) + bilinear = np.zeros([f_shape[0], f_shape[1], f_shape[2]]) + for x in range(width): + for y in range(heigh): + for z in range(depth): + value = (1 - abs(x / f - c)) * (1 - abs(y / f - c)) * (1 - abs(z / f - c)) + bilinear[x, y, z] = value + weights = np.zeros(f_shape) + for i in range(f_shape[3]): + weights[:, :, :, i, i] = bilinear + + init = tf.constant_initializer(value=weights, + dtype=tf.float32) + W = vs.variable(name="up_filter", initializer=init, + shape=weights.shape, trainable=trainable, + restore=restore) + tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, W) + return W + + weights = get_deconv_filter(filter_size) + deconv = tf.nn.conv3d_transpose(incoming, weights, output_shape, + strides=strides, padding='SAME') + + deconv.scope = scope + + # Track output tensor. + tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, deconv) + + return deconv + def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', activation='linear', bias=True, weights_init='uniform_scaling', From 09896605bbeff406e9f36aa22b096ad626583936 Mon Sep 17 00:00:00 2001 From: Aymeric Damien Date: Tue, 9 Jan 2018 19:09:07 +0000 Subject: [PATCH 29/61] [WIP] new estimators (#994) --- ACKNOWLEDGMENTS | 37 ++- examples/basics/kmeans.py | 26 ++ examples/basics/random_forest.py | 24 ++ tflearn/__init__.py | 1 + tflearn/data_flow.py | 44 ++- tflearn/data_utils.py | 43 ++- tflearn/distances.py | 23 ++ tflearn/estimators/__init__.py | 2 + tflearn/estimators/base.py | 147 ++++++++ tflearn/estimators/cluster/__init__.py | 1 + tflearn/estimators/cluster/kmeans.py | 421 +++++++++++++++++++++++ tflearn/estimators/ensemble/__init__.py | 1 + tflearn/estimators/ensemble/forest.py | 424 ++++++++++++++++++++++++ tflearn/initializations.py | 2 +- tflearn/metrics.py | 2 +- tflearn/utils.py | 67 ++++ 16 files changed, 1258 insertions(+), 7 deletions(-) create mode 100644 examples/basics/kmeans.py create mode 100644 examples/basics/random_forest.py create mode 100644 tflearn/distances.py create mode 100644 tflearn/estimators/__init__.py create mode 100644 tflearn/estimators/base.py create mode 100644 tflearn/estimators/cluster/__init__.py create mode 100644 tflearn/estimators/cluster/kmeans.py create mode 100644 tflearn/estimators/ensemble/__init__.py create mode 100644 tflearn/estimators/ensemble/forest.py diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS index 2624e1f4..6a5a6167 100644 --- a/ACKNOWLEDGMENTS +++ b/ACKNOWLEDGMENTS @@ -2,7 +2,8 @@ TFLearn was created to provide a new transparent and simple interface over Tenso - [TensorFlow](http://tensorflow.org) for some derivated code (copyright below). As a higher-level API, TFLearn is heavily relying on TensorFlow base API. - [TensorFlow models](https://github.com/tensorflow/models). -- [Lasagne](https://github.com/Lasagne/Lasagne) (MIT License) TFLearn model building concept with layers is directly inspired from Lasagne. While Lasagne only support Theano, TFLearn is aiming at providing similar concept over TensorFlow instead. +- [SKLearn](http://scikit-learn.org) (NEW BSD License) TFLearn estimators are an adaptation of SKLearn in TensorFLow, as such, TFLearn reuses some structure and documentation from it. +- [Lasagne](https://github.com/Lasagne/Lasagne) (MIT License) Originally, TFLearn neural network building concept is directly inspired from Lasagne. While Lasagne only support Theano, TFLearn is aiming at providing similar concept over TensorFlow instead. - [Keras](http://keras.io) (MIT License) A few layers structure. # TensorFlow is subject to the following copyright notice: @@ -210,3 +211,37 @@ Copyright 2015 The TensorFlow Authors. All rights reserved. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +# SKLearn is subject to the following license: + +New BSD License + +Copyright (c) 2007–2017 The scikit-learn developers. +All rights reserved. + + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of the Scikit-learn Developers nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/examples/basics/kmeans.py b/examples/basics/kmeans.py new file mode 100644 index 00000000..15fb1a4f --- /dev/null +++ b/examples/basics/kmeans.py @@ -0,0 +1,26 @@ +""" K-Means Example """ + +from __future__ import division, print_function, absolute_import + +from tflearn.estimators import KMeans + +# Data loading and preprocessing +import tflearn.datasets.mnist as mnist +X, Y, testX, testY = mnist.load_data(one_hot=False) + +# K-Means training +m = KMeans(n_clusters=10, distance='squared_euclidean') +m.fit(X, display_step=10) + +# Testing +print("Clusters center coordinates:") +print(m.cluster_centers_vars) + +print("X[0] nearest cluster:") +print(m.labels_[0]) + +print("Predicting testX[0] nearest cluster:") +print(m.predict(testX[0])) + +print("Transforming testX[0] to a cluster-distance space:") +print(m.transform(testX[0])) diff --git a/examples/basics/random_forest.py b/examples/basics/random_forest.py new file mode 100644 index 00000000..3e2a5ff6 --- /dev/null +++ b/examples/basics/random_forest.py @@ -0,0 +1,24 @@ +""" Random Forest example. """ + +from __future__ import division, print_function, absolute_import + +import tflearn +from tflearn.estimators import RandomForestClassifier + +# Data loading and pre-processing +import tflearn.datasets.mnist as mnist +X, Y, testX, testY = mnist.load_data(one_hot=False) + +m = RandomForestClassifier(n_estimators=100, max_nodes=1000) +m.fit(X, Y, batch_size=10000, display_step=10) + +print("Compute the accuracy on train set:") +print(m.evaluate(X, Y, tflearn.accuracy_op)) + +print("Compute the accuracy on test set:") +print(m.evaluate(testX, testY, tflearn.accuracy_op)) + +print("Digits for test images id 0 to 5:") +print(m.predict(testX[:5])) +print("True digits:") +print(testY[:5]) diff --git a/tflearn/__init__.py b/tflearn/__init__.py index 80fd9635..11de3fe1 100644 --- a/tflearn/__init__.py +++ b/tflearn/__init__.py @@ -21,6 +21,7 @@ from .layers import normalization from . import metrics from . import activations +from . import distances from . import losses from . import initializations from . import optimizers diff --git a/tflearn/data_flow.py b/tflearn/data_flow.py index 36377493..26db23de 100644 --- a/tflearn/data_flow.py +++ b/tflearn/data_flow.py @@ -305,10 +305,13 @@ class ArrayFlow(object): multi_inputs is True. """ - def __init__(self, X, Y, multi_inputs=False, batch_size=32, shuffle=True): + def __init__(self, X, Y, multi_inputs=False, batch_size=32, shuffle=True, + capacity=None): # Handle multiple inputs if not multi_inputs: X = [X] + if not capacity: + capacity =batch_size * 8 X = [np.array(x) for x in X] self.X = X self.Xlen = len(X[0]) @@ -326,7 +329,7 @@ def __init__(self, X, Y, multi_inputs=False, batch_size=32, shuffle=True): # FIFO Queue for feeding data self.queue = tf.FIFOQueue( dtypes=[x.dtype for x in self.tensorX] + [self.tensorY.dtype], - capacity=batch_size * 8) + capacity=capacity) self.enqueue_op = self.queue.enqueue(self.tensorX + [self.tensorY]) self.batch_size = batch_size self.multi_inputs = multi_inputs @@ -369,3 +372,40 @@ def thread_main(self, sess): for i, x in enumerate(self.tensorX): feed_dict[x] = dataX[i] sess.run(self.enqueue_op, feed_dict=feed_dict) + + +def generate_data_tensor(X, Y, batch_size, shuffle=True, num_threads=1, + capacity=None): + #TODO: Add a way with no batch? + #TODO: Set threads to #CPUs fo machine + cr = None + if capacity is None: + capacity = batch_size * num_threads * 4 + + if isinstance(X, tf.Tensor) and isinstance(Y, tf.Tensor): + # Optional Image and Label Batching + if shuffle: + X, Y = tf.train.shuffle_batch([X, Y], batch_size=batch_size, + min_after_dequeue=batch_size, + capacity=capacity, + num_threads=num_threads) + else: + X, Y = tf.train.batch([X, Y], batch_size=batch_size, + capacity=capacity, + num_threads=num_threads) + + # Array Input + elif X is not None and Y is not None: + X_shape = list(np.shape(X)) + Y_shape = list(np.shape(Y)) + # Create a queue using feed_dicts + cr = ArrayFlow(X, Y, batch_size=batch_size, shuffle=shuffle, + capacity=capacity) + X, Y = cr.get() + # Assign a shape to tensors + X_reshape = [-1] + X_shape[1:] if len(X_shape[1:]) > 0 else [-1, 1] + Y_reshape = [-1] + Y_shape[1:] if len(Y_shape[1:]) > 0 else [-1, 1] + X = tf.reshape(X, X_reshape) + Y = tf.reshape(Y, Y_reshape) + + return X, Y, cr diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 89b726e1..c3e93a41 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -8,6 +8,7 @@ import pickle import csv import warnings +import tensorflow as tf try: #py3 from urllib.parse import urlparse from urllib import request @@ -16,7 +17,6 @@ from six.moves.urllib import request from io import BytesIO - """ Preprocessing provides some useful functions to preprocess data before training, such as pictures dataset building, sequence padding, etc... @@ -43,7 +43,7 @@ def to_categorical(y, nb_classes=None): y: `array`. Class vector to convert. nb_classes: `unused`. Used for older code compatibility. """ - return (y[:, None] == np.unique(y)).astype(np.float32) + y[:, None] == np.unique(y)).astype(np.float32) # ===================== @@ -419,6 +419,7 @@ class containing the images to classify. else: dataset['Y'][i] = labels[i] + def get_img_channel(image_path): """ Load a image and return the channel of the image @@ -433,6 +434,7 @@ def get_img_channel(image_path): channel = 1 return channel + def image_preloader(target_path, image_shape, mode='file', normalize=True, grayscale=False, categorical_labels=True, files_extension=None, filter_channel=False): @@ -857,6 +859,43 @@ def preload(self, label, n_class, categorical_label): return label +def is_array(X): + return type(X) in [np.array, np.ndarray, list] + + +def get_num_features(X): + if isinstance(X, tf.Tensor): + return X.get_shape().as_list()[-1] + elif is_array(X): + return list(np.shape(X))[-1] + else: + raise ValueError("Unknown data type.") + + +def get_num_classes(Y): + if is_array(Y): + # Assume max integer is number of classes + return np.max(Y) + 1 + elif isinstance(Y, tf.Tensor): + return ValueError("Cannot automatically retrieve number of classes " + "from a Tensor. Please fill 'num_classes' argument.") + else: + raise ValueError("Unknown data type.") + + +def get_num_sample(X): + if is_array(X): + return np.shape(X)[0] + elif isinstance(X, tf.Tensor): + return X.get_shape()[0] + else: + raise ValueError("Unknown data type.") + + +# ================== +# STATS UTILS +# ================== + def get_max(X): return np.max(X) diff --git a/tflearn/distances.py b/tflearn/distances.py new file mode 100644 index 00000000..6b34ea0a --- /dev/null +++ b/tflearn/distances.py @@ -0,0 +1,23 @@ +""" Distance Ops """ + +from __future__ import division, print_function, absolute_import + +import tensorflow as tf + +from .utils import get_from_module + + +def get(identifier): + if hasattr(identifier, '__call__'): + return identifier + else: + return get_from_module(identifier, globals(), 'distances') + + +def euclidean(a, b): + return tf.sqrt(tf.reduce_sum(tf.square(a - b), + reduction_indices=0)) + + +def cosine(a, b): + return 1 - tf.matmul(a, b) diff --git a/tflearn/estimators/__init__.py b/tflearn/estimators/__init__.py new file mode 100644 index 00000000..e86918e1 --- /dev/null +++ b/tflearn/estimators/__init__.py @@ -0,0 +1,2 @@ +from .ensemble import RandomForestRegressor, RandomForestClassifier +from .cluster import KMeans, MiniBatchKMeans diff --git a/tflearn/estimators/base.py b/tflearn/estimators/base.py new file mode 100644 index 00000000..919c0ff5 --- /dev/null +++ b/tflearn/estimators/base.py @@ -0,0 +1,147 @@ +from __future__ import division, print_function, absolute_import + +import os +import tensorflow as tf +from tensorflow.python.ops import resources + +from ..utils import validate_func + + +class GraphBranch(object): + """ A graph branch class used for building part of an Estimator graph. + """ + def __init__(self, input_tensor=None, output_tensor=None, params=None): + self.input_tensor = input_tensor + self.output_tensor = output_tensor + self.params = params if params is not None else dict() + self._is_ready = False + if input_tensor is not None and output_tensor is not None: + self._is_ready = True + + def build(self, input_tensor, output_tensor, params=None): + self.input_tensor = input_tensor + self.output_tensor = output_tensor + self.params = params if params is not None else dict() + self._is_ready = True + + @property + def is_ready(self): + return self._is_ready + + def get_params(self, x): + if x in self.params.keys(): + return self.params[x] + else: + return None + + +class BaseEstimator(object): + + """ Estimators Graph is only build when fit/predict or evaluate is called. + """ + + def __init__(self, metric=None, log_dir='/tmp/tflearn_logs/', + global_step=None, session=None, graph=None, name=None): + + self.name = name + + # Estimator Graph and Session + self.graph = tf.Graph() if graph is None else graph + with self.graph.as_default(): + conf = tf.ConfigProto(allow_soft_placement=True) + self.session = tf.Session(config=conf) if session is None else session + if global_step is None: + with self.graph.as_default(): + self.global_step = tf.train.get_or_create_global_step() + + self.metric = validate_func(metric) + + # Estimator Graph Branches + self._train = GraphBranch() + self._pred = GraphBranch() + self._transform = GraphBranch() + self._eval = GraphBranch() + + # Tensor Utils + if not os.path.exists(log_dir): + os.makedirs(log_dir) + self.log_dir = log_dir + self._is_initialized = False + self._to_be_restored = False + + # Ops + self.train_op = None + self.loss_op = None + + # ----------------- + # Initializations + # ----------------- + def _init_graph(self): + # Initialize all weights + if not self._is_initialized: + self.saver = tf.train.Saver() + init_vars = tf.group(tf.global_variables_initializer(), + resources.initialize_resources( + resources.shared_resources())) + self.session.run(init_vars) + self._is_initialized = True + # Restore weights if needed + if self._to_be_restored: + self.saver = tf.train.Saver() + self.saver.restore(self.session, self._to_be_restored) + self._to_be_restored = False + + def _init_estimator(self): + raise NotImplementedError + + # ---------------------- + # Build Graph Branches + # ---------------------- + def _build_fit(self, X, Y, batch_size, multi_inputs=False): + if not self._train._is_ready: + self._init_graph() + train_params = {'X': X, 'Y': Y, 'batch_size': batch_size, + 'multi_inputs': multi_inputs} + self._train.build(None, None, train_params) + + def _build_pred(self, input_tensor, output_tensor): + self._pred.build(input_tensor, output_tensor) + + def _build_transform(self, input_tensor, output_tensor): + self._transform.build(input_tensor, output_tensor) + + def _build_eval(self, X, Y, metric, batch_size, multi_inputs=False): + eval_params = {'X': X, 'Y': Y, 'batch_size': batch_size, + 'metric': metric, 'multi_inputs': multi_inputs} + self._eval.build(None, None, eval_params) + + # --------- + # Methods + # --------- + def fit(self, *args): + #TODO: Handle multiple fits + raise NotImplementedError + + def predict(self, *args): + raise NotImplementedError + + def evaluate(self, *args): + raise NotImplementedError + + def load(self, *args): + raise NotImplementedError + + def save(self, *args): + raise NotImplementedError + + +class SupervisedEstimator(BaseEstimator): + + def __init__(self, metric=None, log_dir='/tmp/tflearn_logs/', + global_step=None, session=None, graph=None, name=None): + super(SupervisedEstimator, self).__init__( + metric=metric, log_dir=log_dir, global_step=global_step, + session=session, graph=graph, name=name) + + def fit(self, X, Y, *args): + pass diff --git a/tflearn/estimators/cluster/__init__.py b/tflearn/estimators/cluster/__init__.py new file mode 100644 index 00000000..e78ec7d8 --- /dev/null +++ b/tflearn/estimators/cluster/__init__.py @@ -0,0 +1 @@ +from .kmeans import KMeans, MiniBatchKMeans diff --git a/tflearn/estimators/cluster/kmeans.py b/tflearn/estimators/cluster/kmeans.py new file mode 100644 index 00000000..25931a1d --- /dev/null +++ b/tflearn/estimators/cluster/kmeans.py @@ -0,0 +1,421 @@ +from __future__ import division, print_function, absolute_import + +from datetime import datetime +import os +import math +import numpy as np +import time + +import tensorflow as tf +from tensorflow.contrib.factorization.python.ops import clustering_ops as c_ops +from tensorflow.contrib.tensor_forest.python.ops import data_ops +from tensorflow.python.ops import state_ops, array_ops, math_ops + +from ...utils import validate_dim, read_tensor_in_checkpoint, prepare_X +from ...data_utils import get_num_features, get_num_sample +from ...data_flow import generate_data_tensor +from ...distances import euclidean, cosine + +from ..base import BaseEstimator + + +class KMeansBase(BaseEstimator): + + def __init__(self, n_clusters, max_iter=300, init=c_ops.RANDOM_INIT, + distance=c_ops.SQUARED_EUCLIDEAN_DISTANCE, + metric=None, num_features=None, log_dir='/tmp/tflearn_logs/', + global_step=None, session=None, graph=None, name=None): + super(KMeansBase, self).__init__( + metric=metric, log_dir=log_dir, global_step=global_step, + session=session, graph=graph, name=name) + + self._estimator_built = False + + # Params + self.n_clusters = n_clusters + self.max_iter = max_iter + self.init = init + self.distance = distance + self.num_features = num_features + self.use_mini_batch = False + + def _build_estimator(self, X=None): + + if not self._estimator_built: + + if self.num_features is None: + self.num_features = get_num_features(X) + + # Reload params from checkpoint if available + if self._to_be_restored and self.num_features is None: + self.num_features = read_tensor_in_checkpoint( + 'num_features', self._to_be_restored) + if self._to_be_restored and self.num_classes is None: + self.num_classes = read_tensor_in_checkpoint( + 'num_classes', self._to_be_restored) + + # Purity checks + if self.num_features is None: + raise ValueError("'num_features' cannot be None.") + + # Persistent Parameters + tf.Variable(self.num_features, dtype=tf.int32, name='num_features') + + self._kmeans = c_ops.KMeans(X, self.n_clusters, + initial_clusters=self.init, + distance_metric=self.distance, + use_mini_batch=self.use_mini_batch) + (self._all_scores, self._cluster_idx, self._scores, + self._cluster_centers_initialized, self._cluster_centers_vars, + self._init_op, self._train_op) = self._kmeans.training_graph() + + # fix for cluster_idx being a tuple + self._cluster_idx = self._cluster_idx[0] + self.avg_distance = tf.reduce_mean(self._scores) + + self._estimator_built = True + self._init_graph() + + @property + def cluster_centers_vars(self): + if self._estimator_built: + return self.session.run(self._cluster_centers_vars) + else: + return None + + @property + def cluster_idx(self): + if self._estimator_built: + return self.session.run(self._cluster_idx) + else: + return None + + @property + def scores(self): + if self._estimator_built: + return self.session.run(self._cluster_centers_vars) + else: + return None + + @property + def all_scores(self): + if self._estimator_built: + return self.session.run(self._cluster_centers_vars) + else: + return None + + # SKLearn bindings + @property + def cluster_centers_(self): + """ Coordinates of cluster centers. """ + return self.cluster_centers_vars + + @property + def labels_(self): + """ Labels of each point. """ + return self.cluster_idx + + @property + def distances_(self): + """ Distances of each point to its closest cluster center. """ + return self.session.run(self._scores) + + @property + def all_distances_(self): + """ Distances of each point to each cluster center. """ + return self.session.run(self._all_scores) + + def _init_graph(self): + super(KMeansBase, self)._init_graph() + # Initialize the kmeans op + self.session.run(self._init_op) + + def fit(self, X, shuffle=True, display_step=500, + n_jobs=1, max_steps=None, verbose=0, **kwargs): + + with self.graph.as_default(): + + # Verify data dimension + validate_dim(X, max_dim=2, min_dim=2, var_name='X') + + # Get data size + num_samples = get_num_sample(X) + + # Set batch size + if 'batch_size' in kwargs.keys(): + batch_size = kwargs['batch_size'] + else: + batch_size = num_samples + + # Build Tree Graph + self._build_estimator(X) + + # Generate Data Tensors. Be aware that every fit with different + # data will re-create a data tensor. + if self._train.get_params('X') != hex(id(X)) or \ + self._train.get_params('batch_size') != batch_size or \ + not self._train.is_ready: + + #TODO: raise Exception("Fitting different data not supported") + + X, _, cr = generate_data_tensor(X, X, batch_size=batch_size, + shuffle=shuffle, + num_threads=8) + X, _, spec = data_ops.ParseDataTensorOrDict(X) + + self._train_op = tf.group( + self._train_op, + state_ops.assign_add(self.global_step, 1)) + self._loss_op = self.avg_distance + self._build_fit(X, X, batch_size) + + # Start QueueRunners + tf.train.start_queue_runners(sess=self.session) + if cr: cr.launch_threads(self.session) + + gstep = self.global_step.eval(session=self.session) + + last_loss = [] + loss_val = None + step = 0 + + # Set step to -1 to exit training + while True: + # Monitor loss + if loss_val: last_loss.append(loss_val) + if len(last_loss) > 10: last_loss.pop(0) + + start_time = time.time() + if (step) % display_step == 0: + _, loss_val, idx = self.session.run( + [self._train_op, self._loss_op, self._cluster_idx]) + else: + _, loss_val, idx = self.session.run([self._train_op, + self._loss_op, + self._cluster_idx]) + duration = time.time() - start_time + + if (step) % display_step == 0: + examples_per_sec = batch_size / duration + sec_per_batch = duration + if self.metric: + format_str = '%s: step %d, loss = %.2f, acc = %.2f, ' \ + '(%.1f examples/sec; %.3f sec/batch)' + print(format_str % ( + datetime.now(), step + gstep, loss_val, + examples_per_sec, sec_per_batch)) + else: + format_str = '%s: step %d, loss = %.2f, ' \ + '(%.1f examples/sec; %.3f sec/batch)' + print(format_str % ( + datetime.now(), step + gstep, loss_val, + examples_per_sec, sec_per_batch)) + + step += 1 + + # Automatic stop after ten flat loss + # TODO(aymeric): better stopping. + if len(last_loss) == 10 and np.var(last_loss) <= 0.01 and not max_steps: + break + + # Max Steps stop + if max_steps: + if step == max_steps: + break + + # save_path = os.path.join(self.log_dir, 'kmeans.ckpt') + # self.saver.save(sess=self.session, + # save_path=save_path, + # global_step=self.global_step) + + # ------------ + # Prediction + # ------------ + + def predict(self, X, with_distances=False): + """ predict. + + Predict the closest cluster. + + Arguments: + X: `1-D Array` or `2-D Array` of shape (n_samples, n_features). + The sample(s) to predict. + + Return: + cluster_indices or (cluster_indices, distances). + + """ + + X, orig_ndim = prepare_X(X, 2, max_dim=2, min_dim=1, debug_msg="X") + + with self.graph.as_default(): + # Build Tree Graph + self._build_estimator() + if not self._pred.is_ready: + input = tf.placeholder(tf.float32, name='pred_input', + shape=[None, self.num_features]) + output = c_ops.nearest_neighbors( + input, self._cluster_centers_vars, k=1) + self._build_pred(input, output) + indices, distances = self.session.run(self._pred.output_tensor, + feed_dict={self._pred.input_tensor: X}) + indices = indices[0] + distances = distances[0] + if orig_ndim == 1: + indices = indices[0] + distances = distances[0] + if with_distances: + return indices, distances + return indices + + def transform(self, X): + """ transform. + + Transform X to a cluster-distance space. + + Arguments: + X: `Array` or `list` of `Array`. The sample(s) to transform. + + Returns: + `Array` of shape (n_clusters). The distance of X to each centroid. + + """ + + X, orig_ndim = prepare_X(X, 2, max_dim=2, min_dim=1, debug_msg="X") + + with self.graph.as_default(): + # Build Tree Graph + self._build_estimator() + if not self._transform.is_ready: + input = tf.placeholder(tf.float32, name='transform_input', + shape=[None, self.num_features]) + centers = self._cluster_centers_vars + centers = tf.reshape(centers, shape=[self.n_clusters, + self.num_features]) + + if self.distance == c_ops.SQUARED_EUCLIDEAN_DISTANCE: + dist_fn = euclidean + elif self.distance == c_ops.COSINE_DISTANCE: + dist_fn = cosine + else: + raise Exception("Incorrect distance metric.") + + output = tf.map_fn( + lambda x: tf.map_fn( + lambda y: dist_fn(x, y), + centers), + input) + + self._build_transform(input, output) + distances = self.session.run(self._transform.output_tensor, + feed_dict={self._transform.input_tensor: X}) + if orig_ndim == 1: + distances = distances[0] + return distances + + def save(self, save_path): + """ save. + + Save model to the given path. + + Args: + save_path: `str`. The path to save the model. + + """ + if not self._estimator_built: + with self.graph.as_default(): + self._build_estimator() + self.saver.save(self.session, os.path.abspath(save_path)) + + def load(self, load_path): + """ load. + + Restore model from the given path. + + Args: + load_path: `str`. The model path. + + """ + with self.graph.as_default(): + self.session = tf.Session() + if self._estimator_built: + self.saver.restore(self.session, os.path.abspath(load_path)) + else: + self._to_be_restored = os.path.abspath(load_path) + + +class KMeans(KMeansBase): + """ KMeans. + + K-Means clustering algorithm. + + """ + + def __init__(self, n_clusters, max_iter=300, init=c_ops.RANDOM_INIT, + distance=c_ops.SQUARED_EUCLIDEAN_DISTANCE, + metric=None, num_features=None, log_dir='/tmp/tflearn_logs/', + global_step=None, session=None, graph=None, name=None): + super(KMeans, self).__init__( + n_clusters, max_iter=max_iter, init=init, distance=distance, + metric=metric, num_features=num_features, log_dir=log_dir, + global_step=global_step, session=session, graph=graph, + name=name) + + def fit(self, X, shuffle=True, display_step=500, n_jobs=1, + max_steps=None): + """ fit. + + Compute the K-Means clustering for the input data. + + Arguments: + X: `Array` or `list` of `Array` of shape (n_samples, n_features). + The training data. + shuffle: `bool`. If True, data are shuffled. + display_step: `int`. The step to display training information. + n_jobs: `int`. The number of jobs to use for the computation. + max_steps: `int`. Maximum number of optimization steps to run. + + """ + + super(KMeans, self).fit(X, shuffle=shuffle, display_step=display_step, + n_jobs=n_jobs, max_steps=max_steps) + + +class MiniBatchKMeans(KMeans): + """ MiniBatchKMeans. + + K-Means clustering algorithm with mini batch. + + """ + + def __init__(self, n_clusters, max_iter=300, init=c_ops.RANDOM_INIT, + distance=c_ops.SQUARED_EUCLIDEAN_DISTANCE, + metric=None, num_features=None, log_dir='/tmp/tflearn_logs/', + global_step=None, session=None, graph=None, name=None): + super(MiniBatchKMeans, self).__init__( + n_clusters, max_iter=max_iter, init=init, distance=distance, + metric=metric, num_features=num_features, log_dir=log_dir, + global_step=global_step, session=session, graph=graph, + name=name) + + self.use_mini_batch = True + + def fit(self, X, batch_size=1024, shuffle=True, display_step=500, + n_jobs=1, max_steps=None): + """ fit. + + Compute the K-Means clustering for the input data. + + Arguments: + X: `Array` or `list` of `Array` of shape (n_samples, n_features). + The training data. + shuffle: `bool`. If True, data are shuffled. + batch_size: `int`. The batch size. + display_step: `int`. The step to display training information. + n_jobs: `int`. The number of jobs to use for the computation. + max_steps: `int`. Maximum number of optimization steps to run. + + """ + super(KMeans, self).fit(X, shuffle=shuffle, display_step=display_step, + n_jobs=n_jobs, max_steps=max_steps, + batch_size=batch_size) diff --git a/tflearn/estimators/ensemble/__init__.py b/tflearn/estimators/ensemble/__init__.py new file mode 100644 index 00000000..0cb78af7 --- /dev/null +++ b/tflearn/estimators/ensemble/__init__.py @@ -0,0 +1 @@ +from .forest import RandomForestClassifier, RandomForestRegressor diff --git a/tflearn/estimators/ensemble/forest.py b/tflearn/estimators/ensemble/forest.py new file mode 100644 index 00000000..cfe3b048 --- /dev/null +++ b/tflearn/estimators/ensemble/forest.py @@ -0,0 +1,424 @@ +from __future__ import division, print_function, absolute_import + +from datetime import datetime +import os +import math +import numpy as np +import time + +import tensorflow as tf +from tensorflow.contrib.tensor_forest.python import tensor_forest +from tensorflow.contrib.tensor_forest.python.ops import data_ops +from tensorflow.python.ops import state_ops, array_ops, math_ops + +from ...utils import validate_dim, read_tensor_in_checkpoint +from ...data_utils import get_num_features, get_num_classes, get_num_sample +from ...data_flow import generate_data_tensor +from ..base import BaseEstimator + + +class ForestEstimator(BaseEstimator): + """ [WIP] ForesEstimator + """ + + def __init__(self, n_estimators=100, max_nodes=10000, + split_after_samples=25, min_samples_split=2, + bagging_fraction=1.0, num_splits_to_consider=0, + feature_bagging_fraction=1.0, max_fertile_nodes=0, + valid_leaf_threshold=1, dominate_method='bootstrap', + dominate_fraction=0.99, regression=False, n_classes=None, + n_features=None, metric=None, log_dir='/tmp/tflearn_logs/', + global_step=None, session=None, graph=None, name=None): + + super(ForestEstimator, self).__init__(metric=metric, + log_dir=log_dir, + global_step=global_step, + session=session, + graph=graph, + name=name) + self._estimator_built = False + + # Tree Params + self.n_estimators = n_estimators + self.max_nodes = max_nodes + self.split_after_samples = split_after_samples + self.min_samples_split = min_samples_split + self.regression = regression + self.n_classes = n_classes + self.n_features = n_features + self.bagging_fraction = bagging_fraction + self.num_splits_to_consider = num_splits_to_consider + self.feature_bagging_fraction = feature_bagging_fraction + self.max_fertile_nodes = max_fertile_nodes + self.valid_leaf_threshold = valid_leaf_threshold + self.dominate_method = dominate_method + self.dominate_fraction = dominate_fraction + + def _build_estimator(self, X=None, Y=None): + + if not self._estimator_built: + if self.n_features is None: + self.n_features = get_num_features(X) + if self.n_classes is None: + if not self.regression: + self.n_classes = get_num_classes(Y) + else: + self.n_classes = get_num_features(Y) + + # Reload params from checkpoint if available + if self._to_be_restored and self.n_features is None: + self.n_features = read_tensor_in_checkpoint( + 'n_features', self._to_be_restored) + if self._to_be_restored and self.n_classes is None: + self.n_classes = read_tensor_in_checkpoint( + 'n_classes', self._to_be_restored) + + # Purity checks + if self.n_classes is None: + raise ValueError("'n_classes' cannot be None.") + if self.n_features is None: + raise ValueError("'n_features' cannot be None.") + + # Persistent Parameters + tf.Variable(self.n_classes, dtype=tf.int32, name='n_classes') + tf.Variable(self.n_features, dtype=tf.int32, name='n_features') + + # Random Forest Parameters + self.params = tensor_forest.ForestHParams( + num_classes=self.n_classes, num_features=self.n_features, + num_trees=self.n_estimators, max_nodes=self.max_nodes, + split_after_samples=self.split_after_samples, + min_split_samples=self.min_samples_split, + regression=self.regression, + bagging_fraction=self.bagging_fraction, + num_splits_to_consider=self.num_splits_to_consider, + feature_bagging_fraction=self.feature_bagging_fraction, + max_fertile_nodes=self.max_fertile_nodes, + valid_leaf_threshold=self.valid_leaf_threshold, + dominate_method=self.dominate_method, + dominate_fraction=self.dominate_fraction).fill() + self.forest_graph = tensor_forest.RandomForestGraphs(self.params) + self._estimator_built = True + + def fit(self, X, Y, batch_size=1024, shuffle=True, display_step=500, + n_jobs=1, max_steps=None): + """ fit. + + Build a forest of trees from the data provided. + + Arguments: + X: `Tensor` or `Tensor list`. The input data. It must be a list of + `Tensor` in case of multiple inputs. + Y: `Tensor`. The labels/targets tensor. + batch_size: `int`. The batch size. + shuffle: ` bool`. If True, data are shuffled. + display_step: `int`. The step to display training information. + n_jobs: `int`. The number of jobs to use for the computation. + max_steps: `int`. Maximum number of optimization steps to run. + + """ + + with self.graph.as_default(): + + # Verify data dimension + validate_dim(X, max_dim=2, min_dim=2, var_name='X') + if not self.regression: + validate_dim(Y, max_dim=1, min_dim=1, var_name='Y') + else: + validate_dim(Y, min_dim=1, var_name='Y') + + # Get data size + num_samples = get_num_sample(X) + + # Build Tree Graph + self._build_estimator(X, Y) + + # Generate Data Tensors. Be aware that every fit with different + # data will re-create a data tensor. + if self._train.get_params('X') != hex(id(X)) or \ + self._train.get_params('Y') != hex(id(Y)) or \ + self._train.get_params('batch_size') != batch_size or \ + not self._train.is_ready: + + X, Y, cr = generate_data_tensor(X, Y, batch_size=batch_size, + shuffle=shuffle, + num_threads=8) + X, _, spec = data_ops.ParseDataTensorOrDict(X) + Y = data_ops.ParseLabelTensorOrDict(Y) + + self._train_op = tf.group( + self.forest_graph.training_graph(X, Y, num_trainers=n_jobs), + state_ops.assign_add(self.global_step, 1)) + self._loss_op = self.forest_graph.training_loss(X, Y) + self._build_fit(X, Y, batch_size) + + # Start QueueRunners + tf.train.start_queue_runners(sess=self.session) + if cr: cr.launch_threads(self.session) + + self._init_graph() + + gstep = self.global_step.eval(session=self.session) + + last_loss = [] + loss_val = None + step = 0 + + # Set step to -1 to exit training + while True: + # Monitor loss + last_loss.append(loss_val) + if len(last_loss) > 10: last_loss.pop(0) + + start_time = time.time() + if (step) % display_step == 0: + _, loss_val = self.session.run( + [self._train_op, self._loss_op]) # TODO: Add acc + else: + _, loss_val = self.session.run([self._train_op, self._loss_op]) + duration = time.time() - start_time + + if (step) % display_step == 0: + examples_per_sec = batch_size / duration + sec_per_batch = duration + if self.metric: + format_str = '%s: step %d, loss = %.2f, acc = %.2f, ' \ + '(%.1f examples/sec; %.3f sec/batch)' + print(format_str % ( + datetime.now(), step + gstep, loss_val, + examples_per_sec, sec_per_batch)) + else: + format_str = '%s: step %d, loss = %.2f, ' \ + '(%.1f examples/sec; %.3f sec/batch)' + print(format_str % ( + datetime.now(), step + gstep, loss_val, + examples_per_sec, sec_per_batch)) + + step += 1 + + # Automatic stop after ten flat loss + if len(last_loss) == 10 and len(set(last_loss)) <= 1 and not max_steps: + break + + # Max Steps stop + if max_steps: + if step == max_steps: + break + + save_path = os.path.join(self.log_dir, 'randomforest.ckpt') + self.saver.save(sess=self.session, + save_path=save_path, + global_step=self.global_step) + + def predict(self, X): + """ predict. + + Predict scores for X. + + Arguments: + X: `1-D Array` or `2-D Array` of shape (n_samples, n_features). + The sample(s) to predict. + + Return: + `Array` or `list` of `Array`. Prediction scores result. + + """ + with self.graph.as_default(): + # Build Tree Graph + self._build_estimator() + if not self._pred.is_ready: + input = tf.placeholder(tf.float32, name='pred_input', + shape=[None, self.n_features]) + output, _, _ = self.forest_graph.inference_graph(input) + self._build_pred(input, output) + return self.session.run(self._pred.output_tensor, + feed_dict={self._pred.input_tensor: X}) + + def evaluate(self, X, Y, metric, batch_size=None): + """ evaluate. + + Evaluate the forest model with the given data and metric. + + Arguments: + X: `2-D Array` of shape (n_samples, n_features). + The input data to evaluate on. + Y: `1-D Array` of shape (n_samples). The labels/targets data. + metric: `func` returning a `Tensor`. The metric function. + batch_size: `int`. If specified, process the data by batch. + + Return: + The metric value. + + """ + + with self.graph.as_default(): + # Verify data dimension + validate_dim(X, max_dim=2, min_dim=2, var_name='X') + if not self.regression: + validate_dim(Y, max_dim=1, min_dim=1, var_name='Y') + else: + validate_dim(Y, min_dim=1, var_name='Y') + + # Get data size + num_samples = get_num_sample(X) + capacity = None + if batch_size is None: + batch_size = num_samples + capacity = 1 + + # Build Tree Graph + self._build_estimator(X, Y) + + # Generate Data Tensors. Be aware that every eval with different + # data will re-create a data tensor. + if self._eval.get_params('X') != hex(id(X)) or \ + self._eval.get_params('Y') != hex(id(Y)) or \ + self._eval.get_params('batch_size') != batch_size or \ + self._eval.get_params('metric') != metric or \ + not self._eval.is_ready: + + X, Y, cr = generate_data_tensor(X, Y, batch_size=batch_size, + shuffle=False, + num_threads=8, + capacity=capacity) + X, _, spec = data_ops.ParseDataTensorOrDict(X) + Y = data_ops.ParseLabelTensorOrDict(Y) + + if not self.params.regression: + Y = math_ops.to_float(array_ops.one_hot(math_ops.to_int64( + array_ops.squeeze(Y)), self.params.n_classes, 1, 0)) + Y = tf.reshape(Y, [-1, self.n_classes]) + + pred, _, _ = self.forest_graph.inference_graph(X) + self._eval_op = metric(pred, Y) + self._build_eval(X, Y, metric, batch_size) + + # Start QueueRunners + tf.train.start_queue_runners(sess=self.session) + if cr: cr.launch_threads(self.session) + + n_batches = int(math.ceil(float(num_samples) / batch_size)) + + m = 0. + for i in range(n_batches): + m += self.session.run(self._eval_op) / n_batches + return m + + def save(self, save_path): + """ save. + + Save model to the given path. + + Args: + path: `str`. The path to save the model. + + """ + if not self._estimator_built: + with self.graph.as_default(): + self._build_estimator() + self.saver.save(self.session, os.path.abspath(save_path)) + + def load(self, load_path): + """ load. + + Restore model from the given path. + + Args: + path: `str`. The model path. + + """ + with self.graph.as_default(): + self.session = tf.Session() + if self._estimator_built: + self.saver.restore(self.session, os.path.abspath(load_path)) + else: + self._to_be_restored = os.path.abspath(load_path) + + +class RandomForestClassifier(ForestEstimator): + """ [WIP] Random Forest Classifier. + + """ + + def __init__(self, n_estimators=10, max_nodes=100, + split_after_samples=25, n_classes=None, n_features=None, + metric=None, log_dir='/tmp/tflearn_logs/', global_step=None, + session=None, graph=None, name=None): + super(RandomForestClassifier, self).__init__( + n_estimators=n_estimators, max_nodes=max_nodes, + split_after_samples=split_after_samples, regression=False, + n_classes=n_classes, n_features=n_features, metric=metric, + log_dir=log_dir, global_step=global_step, session=session, + graph=graph, name=name) + + def predict(self, X): + """ predict. + + Predict class for X. + + Arguments: + X: array-like or sparse matrix of shape = [n_samples, n_features] + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + Returns: + Y: array of shape = [n_samples] or [n_samples, n_outputs] + The predicted classes. + """ + sc = super(RandomForestClassifier, self) + return np.argmax(sc.predict(X), axis=1) + + def predict_proba(self, X): + """ predict_proba. + + Predict class probablities for X. + + Arguments: + X: array-like or sparse matrix of shape = [n_samples, n_features] + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + + Returns: + p : array of shape = [n_samples, n_classes], or a list of n_outputs + such arrays if n_outputs > 1. + The class probabilities of the input samples. The order of the + classes corresponds to that in the attribute `classes_`. + """ + sc = super(RandomForestClassifier, self) + return sc.predict(X) + + def predict_log_proba(self, X): + """ predict_log_proba. + + Predict class log-probabilities for X. + + Arguments: + X: array-like or sparse matrix of shape = [n_samples, n_features] + The input samples. Internally, its dtype will be converted to + ``dtype=np.float32``. If a sparse matrix is provided, it will be + converted into a sparse ``csr_matrix``. + Returns: + p: array of shape = [n_samples, n_classes], or a list of n_outputs + such arrays if n_outputs > 1. + The class probabilities of the input samples. The order of the + classes corresponds to that in the attribute `classes_`. + """ + return np.log(self.predict_proba(X)) + + +class RandomForestRegressor(ForestEstimator): + """ [WIP] Random Forest Regressor. + + """ + + def __init__(self, n_estimators=10, max_nodes=100, + split_after_samples=25, n_features=None, num_output=None, + metric=None, log_dir='/tmp/tflearn_logs/', global_step=None, + session=None, graph=None, name=None): + super(RandomForestRegressor, self).__init__( + n_estimators=n_estimators, max_nodes=max_nodes, + split_after_samples=split_after_samples, regression=True, + n_classes=num_output, n_features=n_features, metric=metric, + log_dir=log_dir, global_step=global_step, session=session, + graph=graph, name=name) diff --git a/tflearn/initializations.py b/tflearn/initializations.py index 1ec03fd1..2f8aa4e2 100644 --- a/tflearn/initializations.py +++ b/tflearn/initializations.py @@ -19,7 +19,7 @@ def get(identifier): if hasattr(identifier, '__call__'): return identifier else: - return get_from_module(identifier, globals(), 'initialization') + return get_from_module(identifier, globals(), 'initializations') def zeros(shape=None, dtype=tf.float32, seed=None): diff --git a/tflearn/metrics.py b/tflearn/metrics.py index 340c293d..39e6b751 100644 --- a/tflearn/metrics.py +++ b/tflearn/metrics.py @@ -5,7 +5,7 @@ def get(identifier): - return get_from_module(identifier, globals(), 'optimizer') + return get_from_module(identifier, globals(), 'metrics') """ Metric classes are meant to be used with TFLearn models (such as DNN). For diff --git a/tflearn/utils.py b/tflearn/utils.py index 44a2aaef..606292f6 100644 --- a/tflearn/utils.py +++ b/tflearn/utils.py @@ -12,6 +12,7 @@ H5PY_SUPPORTED = False import numpy as np import tensorflow as tf +from tensorflow.python import pywrap_tensorflow import tflearn.variables as vs @@ -133,6 +134,22 @@ def get_all_tensor_children(tensor): children_list += get_all_tensor_children(t) return list(set(children_list)) + +# --------------- +# Tensor Utils +# --------------- + +def read_tensor_in_checkpoint(tensor_name, checkpoint_path): + try: + reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_path) + return reader.get_tensor(tensor_name) + except Exception as e: # pylint: disable=broad-except + print(str(e)) + if "corrupted compressed block contents" in str(e): + print("It's likely that your checkpoint file has been compressed " + "with SNAPPY.") + + # ------------------ # Other utils # ------------------ @@ -512,3 +529,53 @@ def fix_saver(collection_lists=None): tf.add_to_collection(tf.GraphKeys.DATA_AUG, t) for t in collection_lists[3]: tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, t) + + +def validate_func(x, allow_none=True): + if not (allow_none and x is None) and hasattr(x, '__call__'): + raise ValueError("'%s' must be a function." % x.__name__) + + +def validate_dim(x, max_dim=None, min_dim=None, var_name='var'): + # Calculate dimension + if isinstance(x, tf.Tensor): + dim = len(x.get_shape().as_list()) + elif type(x) in [np.ndarray, np.array, list]: + dim = np.ndim(x) + else: + #TODO: check hdf5, panda + return + # Verify dimension conditions + if max_dim == min_dim: + if dim != max_dim: + raise ValueError("%s must be %s-D." % (var_name, max_dim)) + else: + if min_dim and dim < min_dim: + raise ValueError( + "%s must be at least %s-D." % (var_name, min_dim)) + elif max_dim and dim > max_dim: + raise ValueError( + "%s must be %s-D or less." % (var_name, max_dim)) + + +def prepare_X(X, target_ndim, max_dim=None, min_dim=None, debug_msg="Data"): + + # Validate the dimension + validate_dim(X, max_dim, min_dim) + + X_ndim = np.ndim(X) + # Reshape to the desired dimension + if X_ndim < target_ndim: + for i in range(target_ndim - X_ndim): + try: + X = np.expand_dims(X, axis=0) + except Exception: + raise Exception(debug_msg + " shape mismatch (too few dimensions).") + elif X_ndim > target_ndim: + for i in range(X_ndim - target_ndim): + try: + X = np.reshape(X, newshape=np.shape(X)[:-1]) + except Exception: + raise Exception(debug_msg + " shape mismatch (too many dimensions).") + return X, X_ndim + From 83d08d1a3e6740d4867588c6570d25c8d40c850c Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Tue, 9 Jan 2018 19:23:08 +0000 Subject: [PATCH 30/61] fix bug --- tflearn/data_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index c3e93a41..9a51bfa5 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -43,7 +43,7 @@ def to_categorical(y, nb_classes=None): y: `array`. Class vector to convert. nb_classes: `unused`. Used for older code compatibility. """ - y[:, None] == np.unique(y)).astype(np.float32) + return (y[:, None] == np.unique(y)).astype(np.float32) # ===================== From 379eeb3e809be856dfe41c88203899f07ea1e4c7 Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Thu, 11 Jan 2018 19:53:26 +0000 Subject: [PATCH 31/61] fix compatibility issues --- examples/images/dcgan.py | 6 +++--- tflearn/data_utils.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/images/dcgan.py b/examples/images/dcgan.py index cddaa519..4cfae763 100644 --- a/examples/images/dcgan.py +++ b/examples/images/dcgan.py @@ -99,15 +99,15 @@ def discriminator(x, reuse=False): # Prepare target data to feed to the discriminator (0: fake image, 1: real image) y_disc_fake = np.zeros(shape=[total_samples]) y_disc_real = np.ones(shape=[total_samples]) -y_disc_fake = tflearn.data_utils.to_categorical(y_disc_fake) -y_disc_real = tflearn.data_utils.to_categorical(y_disc_real) +y_disc_fake = tflearn.data_utils.to_categorical(y_disc_fake, 2) +y_disc_real = tflearn.data_utils.to_categorical(y_disc_real, 2) # Prepare input data to feed to the stacked generator/discriminator gen_noise = np.random.uniform(-1., 1., size=[total_samples, z_dim]) # Prepare target data to feed to the discriminator # Generator tries to fool the discriminator, thus target is 1 (e.g. real images) y_gen = np.ones(shape=[total_samples]) -y_gen = tflearn.data_utils.to_categorical(y_gen) +y_gen = tflearn.data_utils.to_categorical(y_gen, 2) # Start training, feed both noise and real images. gan.fit(X_inputs={'input_gen_noise': gen_noise, diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 9a51bfa5..db8d1379 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -41,9 +41,22 @@ def to_categorical(y, nb_classes=None): Arguments: y: `array`. Class vector to convert. - nb_classes: `unused`. Used for older code compatibility. + nb_classes: `int`. The total number of classes. """ - return (y[:, None] == np.unique(y)).astype(np.float32) + if nb_classes is None: + y = np.asarray(y, dtype='int32') + if len(y.shape) > 2: + print("Warning: data array ndim > 2") + if len(y.shape) > 1: + y = y.reshape(-1) + if not nb_classes: + nb_classes = np.max(y) + 1 + Y = np.zeros((len(y), nb_classes)) + Y[np.arange(len(y)), y] = 1. + return Y + else: + y = np.array(y) + return (y[:, None] == np.unique(y)).astype(np.float32) # ===================== From b2225273d26e70249621d02f5328dbbf8b9b360b Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Thu, 11 Jan 2018 19:55:02 +0000 Subject: [PATCH 32/61] minor fix --- tflearn/data_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index db8d1379..a3922f30 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -43,14 +43,12 @@ def to_categorical(y, nb_classes=None): y: `array`. Class vector to convert. nb_classes: `int`. The total number of classes. """ - if nb_classes is None: + if nb_classes: y = np.asarray(y, dtype='int32') if len(y.shape) > 2: print("Warning: data array ndim > 2") if len(y.shape) > 1: y = y.reshape(-1) - if not nb_classes: - nb_classes = np.max(y) + 1 Y = np.zeros((len(y), nb_classes)) Y[np.arange(len(y)), y] = 1. return Y From 184d753f8fe6ab82a5033f6cbef8edc91b40ca8c Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Mon, 15 Jan 2018 20:10:11 +0000 Subject: [PATCH 33/61] added default parameter for weight (#997) --- tflearn/objectives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/objectives.py b/tflearn/objectives.py index 967a4ed4..c9ed3038 100644 --- a/tflearn/objectives.py +++ b/tflearn/objectives.py @@ -103,7 +103,7 @@ def binary_crossentropy(y_pred, y_true): logits=y_pred, labels=y_true)) -def weighted_crossentropy(y_pred, y_true, weight): +def weighted_crossentropy(y_pred, y_true, weight=1.): """ Weighted Crossentropy. Computes weighted sigmoid cross entropy between y_pred (logits) and y_true From b1a34ae7293ed3e388b079e857ba362028da6e9b Mon Sep 17 00:00:00 2001 From: Ahmet Hamza Emra Date: Tue, 23 Jan 2018 18:27:53 -0600 Subject: [PATCH 34/61] VGG19 Network and weights (#1003) * Create VGG19.py * Update README.md --- examples/README.md | 1 + examples/images/VGG19.py | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 examples/images/VGG19.py diff --git a/examples/README.md b/examples/README.md index fd7a15c0..ac5f8af6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -24,6 +24,7 @@ - [Alexnet](https://github.com/tflearn/tflearn/blob/master/examples/images/alexnet.py). Apply Alexnet to Oxford Flowers 17 classification task. - [VGGNet](https://github.com/tflearn/tflearn/blob/master/examples/images/vgg_network.py). Apply VGG Network to Oxford Flowers 17 classification task. - [VGGNet Finetuning (Fast Training)](https://github.com/tflearn/tflearn/blob/master/examples/images/vgg_network_finetuning.py). Use a pre-trained VGG Network and retrain it on your own data, for fast training. +- [VGG19](https://github.com/AhmetHamzaEmra/tflearn/blob/master/examples/images/VGG19.py). Apply VGG19 Network to ImageNet classification task. - [RNN Pixels](https://github.com/tflearn/tflearn/blob/master/examples/images/rnn_pixels.py). Use RNN (over sequence of pixels) to classify images. - [Highway Network](https://github.com/tflearn/tflearn/blob/master/examples/images/highway_dnn.py). Highway Network implementation for classifying MNIST dataset. - [Highway Convolutional Network](https://github.com/tflearn/tflearn/blob/master/examples/images/convnet_highway_mnist.py). Highway Convolutional Network implementation for classifying MNIST dataset. diff --git a/examples/images/VGG19.py b/examples/images/VGG19.py new file mode 100644 index 00000000..debefbfc --- /dev/null +++ b/examples/images/VGG19.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +""" Very Deep Convolutional Networks for Large-Scale Visual Recognition. +Applying VGG 19-layers convolutional network to Imagenet classification task. +References: + Very Deep Convolutional Networks for Large-Scale Image Recognition. + K. Simonyan, A. Zisserman. arXiv technical report, 2014. +Links: + http://arxiv.org/pdf/1409.1556 +""" + +import tflearn +from tflearn.layers.core import input_data, dropout, fully_connected +from tflearn.layers.conv import conv_2d, max_pool_2d +from tflearn.layers.estimator import regression + + +# Building 'VGG Network' +input_layer = input_data(shape=[None, 224, 224, 3]) + +block1_conv1 = conv_2d(input_layer, 64, 3, activation='relu', name='block1_conv1') +block1_conv2 = conv_2d(block1_conv1, 64, 3, activation='relu', name='block1_conv2') +block1_pool = max_pool_2d(block1_conv2, 2, strides=2, name = 'block1_pool') + +block2_conv1 = conv_2d(block1_pool, 128, 3, activation='relu', name='block2_conv1') +block2_conv2 = conv_2d(block2_conv1, 128, 3, activation='relu', name='block2_conv2') +block2_pool = max_pool_2d(block2_conv2, 2, strides=2, name = 'block2_pool') + +block3_conv1 = conv_2d(block2_pool, 256, 3, activation='relu', name='block3_conv1') +block3_conv2 = conv_2d(block3_conv1, 256, 3, activation='relu', name='block3_conv2') +block3_conv3 = conv_2d(block3_conv2, 256, 3, activation='relu', name='block3_conv3') +block3_conv4 = conv_2d(block3_conv3, 256, 3, activation='relu', name='block3_conv4') +block3_pool = max_pool_2d(block3_conv4, 2, strides=2, name = 'block3_pool') + +block4_conv1 = conv_2d(block3_pool, 512, 3, activation='relu', name='block4_conv1') +block4_conv2 = conv_2d(block4_conv1, 512, 3, activation='relu', name='block4_conv2') +block4_conv3 = conv_2d(block4_conv2, 512, 3, activation='relu', name='block4_conv3') +block4_conv4 = conv_2d(block4_conv3, 512, 3, activation='relu', name='block4_conv4') +block4_pool = max_pool_2d(block4_conv4, 2, strides=2, name = 'block4_pool') + +block5_conv1 = conv_2d(block4_pool, 512, 3, activation='relu', name='block5_conv1') +block5_conv2 = conv_2d(block5_conv1, 512, 3, activation='relu', name='block5_conv2') +block5_conv3 = conv_2d(block5_conv2, 512, 3, activation='relu', name='block5_conv3') +block5_conv4 = conv_2d(block5_conv3, 512, 3, activation='relu', name='block5_conv4') +block4_pool = max_pool_2d(block5_conv4, 2, strides=2, name = 'block4_pool') +flatten_layer = tflearn.layers.core.flatten (block4_pool, name='Flatten') + + +fc1 = fully_connected(flatten_layer, 4096, activation='relu') +dp1 = dropout(fc1, 0.5) +fc2 = fully_connected(dp1, 4096, activation='relu') +dp2 = dropout(fc2, 0.5) + +network = fully_connected(dp2, 1000, activation='rmsprop') + +regression = tflearn.regression(network, optimizer='adam', + loss='categorical_crossentropy', + learning_rate=0.001) + +model = tflearn.DNN(regression, checkpoint_path='vgg19', + tensorboard_dir="./logs") + From 70fb38a297232bec6f3dfb635d9cf1ea3028717f Mon Sep 17 00:00:00 2001 From: Jerome Date: Sun, 11 Feb 2018 10:48:28 +0800 Subject: [PATCH 35/61] Extract method from duplicate code in dnn (#1010) * Renamed tflearn.losses to regularizers. The functions within the renamed module are not losses but regularizers. * Replaced logging.warn with logging.warning. Logging.warn is now deprecated. * Extracted method from duplicate code in dnn.py. --- docs/autodoc.py | 4 +-- tflearn/__init__.py | 2 +- tflearn/helpers/regularizer.py | 4 +-- tflearn/layers/conv.py | 20 +++++++------- tflearn/layers/core.py | 6 ++--- tflearn/layers/recurrent.py | 8 +++--- tflearn/models/dnn.py | 36 +++++++++++--------------- tflearn/{losses.py => regularizers.py} | 0 tflearn/variables.py | 2 +- 9 files changed, 38 insertions(+), 44 deletions(-) rename tflearn/{losses.py => regularizers.py} (100%) diff --git a/docs/autodoc.py b/docs/autodoc.py index d2d98290..a32a5c01 100644 --- a/docs/autodoc.py +++ b/docs/autodoc.py @@ -17,7 +17,7 @@ from tflearn import objectives from tflearn import optimizers from tflearn import data_utils -from tflearn import losses +from tflearn import regularizers from tflearn import summaries from tflearn import utils from tflearn import variables @@ -48,7 +48,7 @@ (objectives, 'tflearn.objectives'), (optimizers, 'tflearn.optimizers'), (data_utils, 'tflearn.data_utils'), - (losses, 'tflearn.losses'), + (regularizers, 'tflearn.regularizers'), (summaries, 'tflearn.summaries'), (variables, 'tflearn.variables'), (utils, 'tflearn.utils'), diff --git a/tflearn/__init__.py b/tflearn/__init__.py index 11de3fe1..4dc5ca26 100644 --- a/tflearn/__init__.py +++ b/tflearn/__init__.py @@ -22,7 +22,7 @@ from . import metrics from . import activations from . import distances -from . import losses +from . import regularizers from . import initializations from . import optimizers from . import summaries diff --git a/tflearn/helpers/regularizer.py b/tflearn/helpers/regularizer.py index dba031ea..27f2db59 100644 --- a/tflearn/helpers/regularizer.py +++ b/tflearn/helpers/regularizer.py @@ -1,7 +1,7 @@ from __future__ import division, print_function, absolute_import import tensorflow as tf -from .. import losses +from .. import regularizers """ @@ -30,7 +30,7 @@ def add_weights_regularizer(variable, loss="L2", weight_decay=0.001, if not add_to_collection: add_to_collection = tf.GraphKeys.REGULARIZATION_LOSSES if isinstance(loss, str): - regul = losses.get(loss) + regul = regularizers.get(loss) weights_regularizer = regul(variable, weight_decay) elif loss and callable(loss): weights_regularizer = loss(variable) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index 6f37c70f..b7b0d12a 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -9,7 +9,7 @@ from .. import variables as vs from .. import activations from .. import initializations -from .. import losses +from .. import regularizers from .. import utils from ..layers.normalization import batch_normalization @@ -81,7 +81,7 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -204,7 +204,7 @@ def conv_2d_transpose(incoming, nb_filter, filter_size, output_shape, filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -358,7 +358,7 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -493,7 +493,7 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -951,7 +951,7 @@ def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -1161,7 +1161,7 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -1283,7 +1283,7 @@ def conv_3d_transpose(incoming, nb_filter, filter_size, output_shape, filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -2033,7 +2033,7 @@ def highway_conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', W_init = initializations.get(weights_init)() W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -2162,7 +2162,7 @@ def highway_conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = vs.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) diff --git a/tflearn/layers/core.py b/tflearn/layers/core.py index f38cac6d..7ff0cab5 100644 --- a/tflearn/layers/core.py +++ b/tflearn/layers/core.py @@ -11,7 +11,7 @@ from tflearn import variables as va from tflearn import activations from tflearn import initializations -from tflearn import losses +from tflearn import regularizers def input_data(shape=None, placeholder=None, dtype=tf.float32, @@ -154,7 +154,7 @@ def fully_connected(incoming, n_units, activation='linear', bias=True, filter_size = None W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = va.variable('W', shape=filter_size, regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) @@ -503,7 +503,7 @@ def highway(incoming, n_units, activation='linear', transform_dropout=None, W_init = initializations.get(weights_init)() W_regul = None if regularizer is not None: - W_regul = lambda x: losses.get(regularizer)(x, weight_decay) + W_regul = lambda x: regularizers.get(regularizer)(x, weight_decay) W = va.variable('W', shape=[n_inputs, n_units], regularizer=W_regul, initializer=W_init, trainable=trainable, restore=restore) diff --git a/tflearn/layers/recurrent.py b/tflearn/layers/recurrent.py index f033a8ba..8a5e43f2 100644 --- a/tflearn/layers/recurrent.py +++ b/tflearn/layers/recurrent.py @@ -412,7 +412,7 @@ def __init__(self, num_units, input_size=None, activation=tf.nn.tanh, bias=True, weights_init=None, trainable=True, restore=True, reuse=False): if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated." % self) + logging.warning("%s: The input_size parameter is deprecated." % self) self._num_units = num_units if isinstance(activation, str): self._activation = activations.get(activation) @@ -471,11 +471,11 @@ def __init__(self, num_units, forget_bias=1.0, input_size=None, inner_activation=tf.sigmoid, bias=True, weights_init=None, trainable=True, restore=True, reuse=False, batch_norm = False): if not state_is_tuple: - logging.warn( + logging.warning( "%s: Using a concatenated state is slower and will soon be " "deprecated. Use state_is_tuple=True." % self) if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated." % self) + logging.warning("%s: The input_size parameter is deprecated." % self) self._num_units = num_units self._forget_bias = forget_bias self._state_is_tuple = state_is_tuple @@ -563,7 +563,7 @@ def __init__(self, num_units, input_size=None, activation=tf.tanh, inner_activation=tf.sigmoid, bias=True, weights_init=None, trainable=True, restore=True, reuse=False): if input_size is not None: - logging.warn("%s: The input_size parameter is deprecated." % self) + logging.warning("%s: The input_size parameter is deprecated." % self) self._num_units = num_units if isinstance(activation, str): self._activation = activations.get(activation) diff --git a/tflearn/models/dnn.py b/tflearn/models/dnn.py index c5c1ccbb..64b95f15 100644 --- a/tflearn/models/dnn.py +++ b/tflearn/models/dnn.py @@ -192,17 +192,7 @@ def fit(self, X_inputs, Y_targets, n_epoch=10, validation_set=None, self.targets) val_feed_dicts = [val_feed_dict for i in self.train_ops] # Retrieve data preprocesing and augmentation - dprep_dict, daug_dict = {}, {} - dprep_collection = tf.get_collection(tf.GraphKeys.DATA_PREP) - daug_collection = tf.get_collection(tf.GraphKeys.DATA_AUG) - for i in range(len(self.inputs)): - # Support for custom inputs not using dprep/daug - if len(dprep_collection) > i: - if dprep_collection[i] is not None: - dprep_dict[self.inputs[i]] = dprep_collection[i] - if len(daug_collection) > i: - if daug_collection[i] is not None: - daug_dict[self.inputs[i]] = daug_collection[i] + daug_dict, dprep_dict = self.retrieve_data_preprocessing_and_augmentation() self.trainer.fit(feed_dicts, val_feed_dicts=val_feed_dicts, n_epoch=n_epoch, show_metric=show_metric, @@ -215,16 +205,7 @@ def fit(self, X_inputs, Y_targets, n_epoch=10, validation_set=None, run_id=run_id, callbacks=callbacks) - def fit_batch(self, X_inputs, Y_targets): - - # For simplicity we build sync dict synchronously but Trainer support - # asynchronous feed dict allocation. - # TODO: check memory impact for large data and multiple optimizers - feed_dict = feed_dict_builder(X_inputs, Y_targets, self.inputs, - self.targets) - feed_dicts = [feed_dict for i in self.train_ops] - - # Retrieve data preprocesing and augmentation + def retrieve_data_preprocessing_and_augmentation(self): dprep_dict, daug_dict = {}, {} dprep_collection = tf.get_collection(tf.GraphKeys.DATA_PREP) daug_collection = tf.get_collection(tf.GraphKeys.DATA_AUG) @@ -236,6 +217,19 @@ def fit_batch(self, X_inputs, Y_targets): if len(daug_collection) > i: if daug_collection[i] is not None: daug_dict[self.inputs[i]] = daug_collection[i] + return daug_dict, dprep_dict + + def fit_batch(self, X_inputs, Y_targets): + + # For simplicity we build sync dict synchronously but Trainer support + # asynchronous feed dict allocation. + # TODO: check memory impact for large data and multiple optimizers + feed_dict = feed_dict_builder(X_inputs, Y_targets, self.inputs, + self.targets) + feed_dicts = [feed_dict for i in self.train_ops] + + # Retrieve data preprocesing and augmentation + daug_dict, dprep_dict = self.retrieve_data_preprocessing_and_augmentation() return self.trainer.fit_batch(feed_dicts, dprep_dict=dprep_dict, daug_dict=daug_dict) diff --git a/tflearn/losses.py b/tflearn/regularizers.py similarity index 100% rename from tflearn/losses.py rename to tflearn/regularizers.py diff --git a/tflearn/variables.py b/tflearn/variables.py index b6dde373..802b5b72 100644 --- a/tflearn/variables.py +++ b/tflearn/variables.py @@ -49,7 +49,7 @@ def variable(name, shape=None, dtype=tf.float32, initializer=None, shape = None if isinstance(regularizer, str): - regularizer = tflearn.losses.get(regularizer) + regularizer = tflearn.regularizers.get(regularizer) collections = set(collections or []) collections |= set([ops.GraphKeys.GLOBAL_VARIABLES, From 81ca9a8a6920e59c5ca1ef98da03f48c46ed1fcd Mon Sep 17 00:00:00 2001 From: Ahmet Hamza Emra Date: Thu, 22 Mar 2018 15:11:57 -0500 Subject: [PATCH 36/61] Tensorflow updated the attribute (#1024) https://www.tensorflow.org/api_docs/python/tf/reduce_mean --- tflearn/objectives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/objectives.py b/tflearn/objectives.py index c9ed3038..859851d4 100644 --- a/tflearn/objectives.py +++ b/tflearn/objectives.py @@ -63,7 +63,7 @@ def categorical_crossentropy(y_pred, y_true): with tf.name_scope("Crossentropy"): y_pred /= tf.reduce_sum(y_pred, reduction_indices=len(y_pred.get_shape())-1, - keep_dims=True) + keepdims=True) # manual computation of crossentropy y_pred = tf.clip_by_value(y_pred, tf.cast(_EPSILON, dtype=_FLOATX), tf.cast(1.-_EPSILON, dtype=_FLOATX)) From f80e4af3c8b7b600d1da62e5260745ceb8018986 Mon Sep 17 00:00:00 2001 From: "Akshay.L.Aradhya" Date: Sat, 31 Mar 2018 12:49:31 +0530 Subject: [PATCH 37/61] Fixing tflearn examples to work with newer versions (#1031) --- examples/extending_tensorflow/layers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/extending_tensorflow/layers.py b/examples/extending_tensorflow/layers.py index a3d7d7d1..e1b8e960 100644 --- a/examples/extending_tensorflow/layers.py +++ b/examples/extending_tensorflow/layers.py @@ -39,25 +39,25 @@ net = tflearn.fully_connected(net, 10, activation='linear') # Defining other ops using Tensorflow - loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(net, Y)) + loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=net, labels=Y)) optimizer = tf.train.AdamOptimizer(learning_rate=0.01).minimize(loss) # Initializing the variables - init = tf.initialize_all_variables() + init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: sess.run(init) batch_size = 128 - for epoch in range(2): # 2 epochs + for epoch in range(2): # 2 epochs avg_cost = 0. - total_batch = int(mnist_data.train.num_examples/batch_size) + total_batch = int(mnist_data.train.num_examples / batch_size) for i in range(total_batch): batch_xs, batch_ys = mnist_data.train.next_batch(batch_size) sess.run(optimizer, feed_dict={X: batch_xs, Y: batch_ys}) cost = sess.run(loss, feed_dict={X: batch_xs, Y: batch_ys}) - avg_cost += cost/total_batch + avg_cost += cost / total_batch if i % 20 == 0: - print("Epoch:", '%03d' % (epoch+1), "Step:", '%03d' % i, + print("Epoch:", '%03d' % (epoch + 1), "Step:", '%03d' % i, "Loss:", str(cost)) From f81fcec2053af23a17f92da98e72cfa280887315 Mon Sep 17 00:00:00 2001 From: Andy1621 <812487273@qq.com> Date: Fri, 1 Jun 2018 10:49:31 +0800 Subject: [PATCH 38/61] Update merge_ops.py (#1050) --- tflearn/layers/merge_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/layers/merge_ops.py b/tflearn/layers/merge_ops.py index ad74800c..65a46e22 100644 --- a/tflearn/layers/merge_ops.py +++ b/tflearn/layers/merge_ops.py @@ -22,7 +22,7 @@ def merge(tensors_list, mode, axis=1, name="Merge"): ``` 'concat': concatenate outputs along specified axis 'elemwise_sum': outputs element-wise sum - 'elemwise_mul': outputs element-wise sum + 'elemwise_mul': outputs element-wise mul 'sum': outputs element-wise sum along specified axis 'mean': outputs element-wise average along specified axis 'prod': outputs element-wise multiplication along specified axis From fead80dd84f803d10ea173c663a55cebb2f04962 Mon Sep 17 00:00:00 2001 From: fraxmans Date: Fri, 1 Jun 2018 10:50:17 +0800 Subject: [PATCH 39/61] fix typo (#1057) --- tflearn/helpers/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/helpers/trainer.py b/tflearn/helpers/trainer.py index 91d41789..746ef3a2 100644 --- a/tflearn/helpers/trainer.py +++ b/tflearn/helpers/trainer.py @@ -286,7 +286,7 @@ def fit(self, feed_dicts, n_epoch=10, val_feed_dicts=None, show_metric=False, daug_dict, show_metric, self.summ_writer, self.coord) - # Prepare TermLogger for training diplay + # Prepare TermLogger for training display metric_term_name = None if train_op.metric is not None: if hasattr(train_op.metric, 'm_name'): From 8aa8436a641f019983a4d5ea2290ba98eed8a181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Csaba=20Kert=C3=A9sz?= Date: Thu, 14 Jun 2018 11:21:44 +0300 Subject: [PATCH 40/61] Fix typo (#1059) --- tflearn/layers/conv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index b7b0d12a..edf94fd0 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -1726,7 +1726,7 @@ def resnext_block(incoming, nb_blocks, out_channels, cardinality, reuse=False, scope=None, name="ResNeXtBlock"): """ ResNeXt Block. - A ResNeXt block as described in ResNeXt paper (Figure 2, c). + A ResNeXt block as described in ResNeXt paper (Figure 3.c). Input: 4-D Tensor [batch, height, width, in_channels]. From 5a674b7f7d70064c811cbd98c4a41a17893d44ee Mon Sep 17 00:00:00 2001 From: ChemicalXandco <32775248+ChemicalXandco@users.noreply.github.com> Date: Mon, 2 Jul 2018 00:25:14 +0100 Subject: [PATCH 41/61] Make exception more informative (#1068) --- tflearn/layers/conv.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index edf94fd0..fdc02d73 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -63,7 +63,7 @@ def conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv2d(filter_size, input_shape[-1], nb_filter) @@ -185,7 +185,7 @@ def conv_2d_transpose(incoming, nb_filter, filter_size, output_shape, """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv2d(filter_size, nb_filter, @@ -341,7 +341,7 @@ def atrous_conv_2d(incoming, nb_filter, filter_size, rate=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv2d(filter_size, input_shape[-1], nb_filter) @@ -472,7 +472,7 @@ def grouped_conv_2d(incoming, channel_multiplier, filter_size, strides=1, """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) nb_filter = channel_multiplier * input_shape[-1] @@ -562,7 +562,7 @@ def max_pool_2d(incoming, kernel_size, strides=None, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_2d(kernel_size) strides = utils.autoformat_kernel_2d(strides) if strides else kernel @@ -607,7 +607,7 @@ def avg_pool_2d(incoming, kernel_size, strides=None, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_2d(kernel_size) strides = utils.autoformat_kernel_2d(strides) if strides else kernel @@ -647,7 +647,7 @@ def upsample_2d(incoming, kernel_size, name="UpSample2D"): """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_2d(kernel_size) with tf.name_scope(name) as scope: @@ -711,7 +711,7 @@ def upscore_layer(incoming, num_classes, shape=None, kernel_size=4, """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) strides = utils.autoformat_kernel_2d(strides) filter_size = utils.autoformat_filter_conv2d(kernel_size, @@ -812,7 +812,7 @@ def upscore_layer3d(incoming, num_classes, shape=None, kernel_size=4, """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D" + assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D, not %d-D" % len(input_shape) strides = utils.autoformat_kernel_3d(strides) filter_size = utils.autoformat_filter_conv3d(kernel_size, @@ -929,7 +929,7 @@ def conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D" + assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv2d(filter_size, input_shape[-1], nb_filter) @@ -1021,7 +1021,7 @@ def max_pool_1d(incoming, kernel_size, strides=None, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D" + assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_2d(kernel_size) kernel = [1, kernel[1], 1, 1] @@ -1070,7 +1070,7 @@ def avg_pool_1d(incoming, kernel_size, strides=None, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D" + assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_2d(kernel_size) kernel = [1, kernel[1], 1, 1] @@ -1143,7 +1143,7 @@ def conv_3d(incoming, nb_filter, filter_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D" + assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv3d(filter_size, input_shape[-1], nb_filter) @@ -1264,7 +1264,7 @@ def conv_3d_transpose(incoming, nb_filter, filter_size, output_shape, """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D" + assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv3d(filter_size, nb_filter, @@ -1366,7 +1366,7 @@ def max_pool_3d(incoming, kernel_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D" + assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_3d(kernel_size) strides = utils.autoformat_stride_3d(strides) @@ -1413,7 +1413,7 @@ def avg_pool_3d(incoming, kernel_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D" + assert len(input_shape) == 5, "Incoming Tensor shape must be 5-D, not %d-D" % len(input_shape) kernel = utils.autoformat_kernel_3d(kernel_size) strides = utils.autoformat_stride_3d(strides) @@ -1449,7 +1449,7 @@ def global_max_pool(incoming, name="GlobalMaxPool"): """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) with tf.name_scope(name): inference = tf.reduce_max(incoming, [1, 2]) @@ -1475,7 +1475,7 @@ def global_avg_pool(incoming, name="GlobalAvgPool"): """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) with tf.name_scope(name): inference = tf.reduce_mean(incoming, [1, 2]) @@ -2016,7 +2016,7 @@ def highway_conv_2d(incoming, nb_filter, filter_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D" + assert len(input_shape) == 4, "Incoming Tensor shape must be 4-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv2d(filter_size, input_shape[-1], nb_filter) @@ -2139,7 +2139,7 @@ def highway_conv_1d(incoming, nb_filter, filter_size, strides=1, padding='same', """ input_shape = utils.get_incoming_shape(incoming) - assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D" + assert len(input_shape) == 3, "Incoming Tensor shape must be 3-D, not %d-D" % len(input_shape) filter_size = utils.autoformat_filter_conv2d(filter_size, input_shape[-1], nb_filter) From 29f08d18f7fdfb451931a92b51add0013ec63a5a Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Mon, 2 Jul 2018 02:28:31 +0300 Subject: [PATCH 42/61] Speed up importing (#1070) * initializations: lazily import xavier and variance scaling from tf.contrib * variables: use vendored copy of tensorflow's add_arg_scope * data_utils: replace VocabularyProcessor with a lazy-loading proxy --- tflearn/data_utils.py | 38 ++++--- tflearn/initializations.py | 21 ++-- tflearn/variables.py | 2 +- tflearn/vendor/__init__.py | 0 tflearn/vendor/arg_scope.py | 216 ++++++++++++++++++++++++++++++++++++ 5 files changed, 246 insertions(+), 31 deletions(-) create mode 100644 tflearn/vendor/__init__.py create mode 100644 tflearn/vendor/arg_scope.py diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index a3922f30..e7ff4f33 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -180,15 +180,8 @@ def random_sequence_from_textfile(path, seq_maxlen): text = open(path).read() return random_sequence_from_string(text, seq_maxlen) -try: - from tensorflow.contrib.learn.python.learn.preprocessing.text import \ - VocabularyProcessor as _VocabularyProcessor -except Exception: - _VocabularyProcessor = object - -# Mirroring TensorFLow `VocabularyProcessor` -class VocabularyProcessor(_VocabularyProcessor): +class VocabularyProcessor(object): """ Vocabulary Processor. Maps documents to sequences of word ids. @@ -209,10 +202,19 @@ def __init__(self, min_frequency=0, vocabulary=None, tokenizer_fn=None): - super(VocabularyProcessor, self).__init__(max_document_length, - min_frequency, - vocabulary, - tokenizer_fn) + from tensorflow.contrib.learn.python.learn.preprocessing.text import \ + VocabularyProcessor as _VocabularyProcessor + self.__dict__['_vocabulary_processor'] = _VocabularyProcessor( + max_document_length, + min_frequency, + vocabulary, + tokenizer_fn) + + def __getattr__(self, key): + return getattr(self._vocabulary_processor, key) + + def __setattr__(self, key, value): + setattr(self._vocabulary_processor, key, value) def fit(self, raw_documents, unused_y=None): """ fit. @@ -226,7 +228,7 @@ def fit(self, raw_documents, unused_y=None): Returns: self """ - return super(VocabularyProcessor, self).fit(raw_documents, unused_y) + return self._vocabulary_processor.fit(raw_documents, unused_y) def fit_transform(self, raw_documents, unused_y=None): """ fit_transform. @@ -240,7 +242,7 @@ def fit_transform(self, raw_documents, unused_y=None): Returns: X: iterable, [n_samples, max_document_length] Word-id matrix. """ - return super(VocabularyProcessor, self).fit_transform(raw_documents, + return self._vocabulary_processor.fit_transform(raw_documents, unused_y) def transform(self, raw_documents): @@ -257,7 +259,7 @@ def transform(self, raw_documents): Yields: X: iterable, [n_samples, max_document_length] Word-id matrix. """ - return super(VocabularyProcessor, self).transform(raw_documents) + return self._vocabulary_processor.transform(raw_documents) def reverse(self, documents): """ reverse. @@ -270,7 +272,7 @@ def reverse(self, documents): Returns: Iterator over mapped in words documents. """ - return super(VocabularyProcessor, self).reverse(documents) + return self._vocabulary_processor.reverse(documents) def save(self, filename): """ save. @@ -280,7 +282,7 @@ def save(self, filename): Arguments: filename: Path to output file. """ - super(VocabularyProcessor, self).save(filename) + return self._vocabulary_processor.save(filename) @classmethod def restore(cls, filename): @@ -294,7 +296,7 @@ def restore(cls, filename): Returns: VocabularyProcessor object. """ - return super(VocabularyProcessor, cls).restore(filename) + return self._vocabulary_processor.restore(filename) # =================== diff --git a/tflearn/initializations.py b/tflearn/initializations.py index 2f8aa4e2..42084656 100644 --- a/tflearn/initializations.py +++ b/tflearn/initializations.py @@ -2,16 +2,7 @@ import math import tensorflow as tf -try: - from tensorflow.contrib.layers.python.layers.initializers import \ - xavier_initializer -except Exception: - xavier_initializer = None -try: - from tensorflow.contrib.layers.python.layers.initializers import \ - variance_scaling_initializer -except Exception: - variance_scaling_initializer = None + from .utils import get_from_module @@ -203,7 +194,10 @@ def xavier(uniform=True, seed=None, dtype=tf.float32): [http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf] (http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf) """ - if xavier_initializer is None: + try: + from tensorflow.contrib.layers.python.layers.initializers import \ + xavier_initializer + except ImportError: raise NotImplementedError("'xavier_initializer' not supported, " "please update TensorFlow.") return xavier_initializer(uniform=uniform, seed=seed, dtype=dtype) @@ -259,7 +253,10 @@ def variance_scaling(factor=2.0, mode='FAN_IN', uniform=False, seed=None, ValueError: if `dtype` is not a floating point type. TypeError: if `mode` is not in ['FAN_IN', 'FAN_OUT', 'FAN_AVG']. """ - if variance_scaling_initializer is None: + try: + from tensorflow.contrib.layers.python.layers.initializers import \ + variance_scaling_initializer + except ImportError: raise NotImplementedError("'variance_scaling_initializer' not " "supported, please update TensorFlow.") return variance_scaling_initializer(factor=factor, mode=mode, diff --git a/tflearn/variables.py b/tflearn/variables.py index 802b5b72..633371b3 100644 --- a/tflearn/variables.py +++ b/tflearn/variables.py @@ -4,7 +4,7 @@ import tensorflow as tf import tflearn -from tensorflow.contrib.framework.python.ops import add_arg_scope as contrib_add_arg_scope +from tflearn.vendor.arg_scope import add_arg_scope as contrib_add_arg_scope from tensorflow.python.framework import ops from tensorflow.python.ops import variable_scope diff --git a/tflearn/vendor/__init__.py b/tflearn/vendor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tflearn/vendor/arg_scope.py b/tflearn/vendor/arg_scope.py new file mode 100644 index 00000000..5f7dd5a5 --- /dev/null +++ b/tflearn/vendor/arg_scope.py @@ -0,0 +1,216 @@ +# This is a vendored copy of +# tensorflow/contrib/framework/python/ops/arg_scope.py at +# tensorflow@4292085f549afc7d7e9ac5dc517b2bab45c79ad3 + +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains the arg_scope used for scoping layers arguments. + + Allows one to define models much more compactly by eliminating boilerplate + code. This is accomplished through the use of argument scoping (arg_scope). + + Example of how to use tf.contrib.framework.arg_scope: + + ``` + from third_party.tensorflow.contrib.layers.python import layers + + arg_scope = tf.contrib.framework.arg_scope + + with arg_scope([layers.conv2d], padding='SAME', + initializer=layers.variance_scaling_initializer(), + regularizer=layers.l2_regularizer(0.05)): + net = layers.conv2d(inputs, 64, [11, 11], 4, padding='VALID', scope='conv1') + net = layers.conv2d(net, 256, [5, 5], scope='conv2') + ``` + The first call to conv2d will behave as follows: + layers.conv2d(inputs, 64, [11, 11], 4, padding='VALID', + initializer=layers.variance_scaling_initializer(), + regularizer=layers.l2_regularizer(0.05), scope='conv1') + + The second call to conv2d will also use the arg_scope's default for padding: + layers.conv2d(inputs, 256, [5, 5], padding='SAME', + initializer=layers.variance_scaling_initializer(), + regularizer=layers.l2_regularizer(0.05), scope='conv2') + + Example of how to reuse an arg_scope: + + ``` + with arg_scope([layers.conv2d], padding='SAME', + initializer=layers.variance_scaling_initializer(), + regularizer=layers.l2_regularizer(0.05)) as sc: + net = layers.conv2d(net, 256, [5, 5], scope='conv1') + .... + + with arg_scope(sc): + net = layers.conv2d(net, 256, [5, 5], scope='conv2') + ``` + + Example of how to use tf.contrib.framework.add_arg_scope to enable your + function to be called within an arg_scope later: + + @tf.contrib.framework.add_arg_scope + def conv2d(*args, **kwargs) +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.util import tf_contextlib +from tensorflow.python.util import tf_decorator + +__all__ = [ + 'arg_scope', 'add_arg_scope', 'current_arg_scope', 'has_arg_scope', + 'arg_scoped_arguments', 'arg_scope_func_key' +] + +_ARGSTACK = [{}] + +_DECORATED_OPS = {} + + +def _get_arg_stack(): + if _ARGSTACK: + return _ARGSTACK + else: + _ARGSTACK.append({}) + return _ARGSTACK + + +def current_arg_scope(): + stack = _get_arg_stack() + return stack[-1] + + +def arg_scope_func_key(op): + return getattr(op, '_key_op', str(op)) + + +def _name_op(op): + return (op.__module__, op.__name__) + + +def _kwarg_names(func): + kwargs_length = len(func.__defaults__) if func.__defaults__ else 0 + return func.__code__.co_varnames[-kwargs_length:func.__code__.co_argcount] + + +def _add_op(op): + key = arg_scope_func_key(op) + if key not in _DECORATED_OPS: + _DECORATED_OPS[key] = _kwarg_names(op) + + +@tf_contextlib.contextmanager +def arg_scope(list_ops_or_scope, **kwargs): + """Stores the default arguments for the given set of list_ops. + + For usage, please see examples at top of the file. + + Args: + list_ops_or_scope: List or tuple of operations to set argument scope for or + a dictionary containing the current scope. When list_ops_or_scope is a + dict, kwargs must be empty. When list_ops_or_scope is a list or tuple, + then every op in it need to be decorated with @add_arg_scope to work. + **kwargs: keyword=value that will define the defaults for each op in + list_ops. All the ops need to accept the given set of arguments. + + Yields: + the current_scope, which is a dictionary of {op: {arg: value}} + Raises: + TypeError: if list_ops is not a list or a tuple. + ValueError: if any op in list_ops has not be decorated with @add_arg_scope. + """ + if isinstance(list_ops_or_scope, dict): + # Assumes that list_ops_or_scope is a scope that is being reused. + if kwargs: + raise ValueError('When attempting to re-use a scope by suppling a' + 'dictionary, kwargs must be empty.') + current_scope = list_ops_or_scope.copy() + try: + _get_arg_stack().append(current_scope) + yield current_scope + finally: + _get_arg_stack().pop() + else: + # Assumes that list_ops_or_scope is a list/tuple of ops with kwargs. + if not isinstance(list_ops_or_scope, (list, tuple)): + raise TypeError('list_ops_or_scope must either be a list/tuple or reused ' + 'scope (i.e. dict)') + try: + current_scope = current_arg_scope().copy() + for op in list_ops_or_scope: + key = arg_scope_func_key(op) + if not has_arg_scope(op): + raise ValueError('%s is not decorated with @add_arg_scope', + _name_op(op)) + if key in current_scope: + current_kwargs = current_scope[key].copy() + current_kwargs.update(kwargs) + current_scope[key] = current_kwargs + else: + current_scope[key] = kwargs.copy() + _get_arg_stack().append(current_scope) + yield current_scope + finally: + _get_arg_stack().pop() + + +def add_arg_scope(func): + """Decorates a function with args so it can be used within an arg_scope. + + Args: + func: function to decorate. + + Returns: + A tuple with the decorated function func_with_args(). + """ + + def func_with_args(*args, **kwargs): + current_scope = current_arg_scope() + current_args = kwargs + key_func = arg_scope_func_key(func) + if key_func in current_scope: + current_args = current_scope[key_func].copy() + current_args.update(kwargs) + return func(*args, **current_args) + + _add_op(func) + setattr(func_with_args, '_key_op', arg_scope_func_key(func)) + return tf_decorator.make_decorator(func, func_with_args) + + +def has_arg_scope(func): + """Checks whether a func has been decorated with @add_arg_scope or not. + + Args: + func: function to check. + + Returns: + a boolean. + """ + return arg_scope_func_key(func) in _DECORATED_OPS + + +def arg_scoped_arguments(func): + """Returns the list kwargs that arg_scope can set for a func. + + Args: + func: function which has been decorated with @add_arg_scope. + + Returns: + a list of kwargs names. + """ + assert has_arg_scope(func) + return _DECORATED_OPS[arg_scope_func_key(func)] From 6934f5c97e9fd4e4af07e240b70a88c21b2f5873 Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Thu, 26 Jul 2018 23:12:33 -0700 Subject: [PATCH 43/61] quick fix --- examples/extending_tensorflow/variables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/extending_tensorflow/variables.py b/examples/extending_tensorflow/variables.py index 89a39796..233905f3 100644 --- a/examples/extending_tensorflow/variables.py +++ b/examples/extending_tensorflow/variables.py @@ -48,7 +48,8 @@ def dnn(x): return x net = dnn(X) - loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(net, Y)) + loss = tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits(logits=net, labels=Y)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) accuracy = tf.reduce_mean( tf.cast(tf.equal(tf.argmax(net, 1), tf.argmax(Y, 1)), tf.float32), From 13b04e4f4a55338a471788390876626aa7d676e3 Mon Sep 17 00:00:00 2001 From: Gershom A Date: Thu, 23 Aug 2018 06:52:22 +0100 Subject: [PATCH 44/61] fixed directory mismatch in cifar10 loaddata (#1084) * fixed directory mismatch in cifar10 loaddata * Update cifar10.py --- tflearn/datasets/cifar10.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tflearn/datasets/cifar10.py b/tflearn/datasets/cifar10.py index fe2282b8..10be161a 100644 --- a/tflearn/datasets/cifar10.py +++ b/tflearn/datasets/cifar10.py @@ -75,10 +75,11 @@ def maybe_download(filename, source_url, work_directory): filepath, reporthook) statinfo = os.stat(filepath) print(('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')) - untar(filepath) + untar(filepath,work_directory) return filepath -#reporthook from stackoverflow #13881092 + +# reporthook from stackoverflow #13881092 def reporthook(blocknum, blocksize, totalsize): readsofar = blocknum * blocksize if totalsize > 0: @@ -86,16 +87,24 @@ def reporthook(blocknum, blocksize, totalsize): s = "\r%5.1f%% %*d / %d" % ( percent, len(str(totalsize)), readsofar, totalsize) sys.stderr.write(s) - if readsofar >= totalsize: # near the end + if readsofar >= totalsize: # near the end sys.stderr.write("\n") - else: # total size is unknown + else: # total size is unknown sys.stderr.write("read %d\n" % (readsofar,)) -def untar(fname): + +def untar(fname,path=""): if (fname.endswith("tar.gz")): tar = tarfile.open(fname) - tar.extractall(path = '/'.join(fname.split('/')[:-1])) + tar.extractall(path=os.path.join( + path, + '/'.join(fname.split('/')[:-1]) + )) tar.close() - print("File Extracted in Current Directory") + if path is "": + print("File Extracted in Current Directory") + else: + print("File Extracted in to ".join(path)) else: print("Not a tar.gz file: '%s '" % sys.argv[0]) + From e532c855542054ac2ee3b376b88d67c80b02fa0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Csaba=20Kert=C3=A9sz?= Date: Mon, 10 Sep 2018 02:08:45 +0300 Subject: [PATCH 45/61] Add support for image base path and float labels in image_preloader() and build_hdf5_image_dataset() (#1086) --- tflearn/data_utils.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index e7ff4f33..bd3d259f 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -306,7 +306,7 @@ def restore(cls, filename): def build_hdf5_image_dataset(target_path, image_shape, output_path='dataset.h5', mode='file', categorical_labels=True, normalize=True, grayscale=False, - files_extension=None, chunks=False): + files_extension=None, chunks=False, image_base_path='', float_labels=False): """ Build HDF5 Image Dataset. Build an HDF5 dataset by providing either a root folder or a plain text @@ -377,6 +377,8 @@ class containing the images to classify. chunks: `bool` Whether to chunks the dataset or not. You should use chunking only when you really need it. See HDF5 documentation. If chunks is 'True' a sensitive default will be computed. + image_base_path: `str`. Base path for the images listed in the file mode. + float_labels: `bool`. Read float labels instead of integers in file mode. """ import h5py @@ -394,8 +396,12 @@ class containing the images to classify. images, labels = [], [] for l in f.readlines(): l = l.strip('\n').split() + l[0] = image_base_path + l[0] images.append(l[0]) - labels.append(int(l[1])) + if float_labels: + labels.append(float(l[1])) + else: + labels.append(int(l[1])) n_classes = np.max(labels) + 1 @@ -450,7 +456,7 @@ def get_img_channel(image_path): def image_preloader(target_path, image_shape, mode='file', normalize=True, grayscale=False, categorical_labels=True, - files_extension=None, filter_channel=False): + files_extension=None, filter_channel=False, image_base_path='', float_labels=False): """ Image PreLoader. Create a python array (`Preloader`) that loads images on the fly (from @@ -519,6 +525,8 @@ class containing the images to classify. all files are allowed. filter_channel: `bool`. If true, images which the channel is not 3 should be filter. + image_base_path: `str`. Base path for the images listed in the file mode. + float_labels: `bool`. Read float labels instead of integers in file mode. Returns: (X, Y): with X the images array and Y the labels array. @@ -533,12 +541,16 @@ class containing the images to classify. images, labels = [], [] for l in f.readlines(): l = l.strip('\n').split() + l[0] = image_base_path + l[0] if not files_extension or any(flag in l[0] for flag in files_extension): if filter_channel: if get_img_channel(l[0]) != 3: continue images.append(l[0]) - labels.append(int(l[1])) + if float_labels: + labels.append(float(l[1])) + else: + labels.append(int(l[1])) n_classes = np.max(labels) + 1 X = ImagePreloader(images, image_shape, normalize, grayscale) From c0baee9d34f41b84dbc43ea28e37baa5dbd465e4 Mon Sep 17 00:00:00 2001 From: angrypark Date: Tue, 25 Sep 2018 12:32:12 +0900 Subject: [PATCH 46/61] fix typo (#1090) "Truncating type '%s' not understood" should have 'truncating', not 'padding' --- tflearn/data_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index bd3d259f..087e1bae 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -100,7 +100,7 @@ def pad_sequences(sequences, maxlen=None, dtype='int32', padding='post', elif truncating == 'post': trunc = s[:maxlen] else: - raise ValueError("Truncating type '%s' not understood" % padding) + raise ValueError("Truncating type '%s' not understood" % truncating) if padding == 'post': x[idx, :len(trunc)] = trunc From fa93b40058f9bfe5a89217d52bb7303d39e770d1 Mon Sep 17 00:00:00 2001 From: Aymeric Damien Date: Tue, 16 Oct 2018 20:49:06 -0700 Subject: [PATCH 47/61] Update SELU (#1092) --- tflearn/activations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tflearn/activations.py b/tflearn/activations.py index 28f61382..1e75ce6b 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -301,6 +301,4 @@ def selu(x): [https://arxiv.org/abs/1706.02515](https://arxiv.org/abs/1706.02515) """ - alpha = 1.6732632423543772848170429916717 - scale = 1.0507009873554804934193349852946 - return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x)) + return tf.nn.selu(x) From cffd677cc30801dbd2dcfd055de481e1741ec6fa Mon Sep 17 00:00:00 2001 From: vishal sharma Date: Fri, 26 Oct 2018 19:54:40 -0600 Subject: [PATCH 48/61] Fixing termlogs for R2 (#1093) * Fixing termlogs for R2 When using R2 as metric, training step displays 'val_acc' instead of 'val_R2' * Fixing termlogs for R2 While using R2 as metric, termlog displays 'val_acc' instead of 'val_R2' --- tflearn/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tflearn/callbacks.py b/tflearn/callbacks.py index 070f163e..3c4b1a07 100644 --- a/tflearn/callbacks.py +++ b/tflearn/callbacks.py @@ -206,7 +206,7 @@ def termlogs(self, step=0, global_loss=None, global_acc=None, step_time=None): if data['val_loss'] is not None: print_val_loss = " | val_loss: " + "%.5f" % data['val_loss'] if data['val_acc'] is not None: - print_val_acc = " - val_acc: " + "%.4f" % data['val_acc'] + print_val_acc = " - val_" + data['metric_name'] + ": " + "%.4f" % data['val_acc'] # fix diplay, if step reached the whole epoch, display epoch - 1, as epoch has been updated print_epoch = data['epoch'] # Smoothing display, so we show display at step + 1 to show data_size/data_size at end From ecbecc84deb0181abaad44aa5fc4e1ee38d6255e Mon Sep 17 00:00:00 2001 From: vishal sharma Date: Fri, 2 Nov 2018 02:42:30 -0600 Subject: [PATCH 49/61] Adding hard sigmoid activation function (#1095) hard sigmoid is faster to compute than sigmoid function --- tflearn/activations.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tflearn/activations.py b/tflearn/activations.py index 1e75ce6b..9c6d2093 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -302,3 +302,22 @@ def selu(x): """ return tf.nn.selu(x) + + +def hard_sigmoid(x): + """Hard sigmoid activation function. + + Segment-wise linear approximation of sigmoid. Faster than sigmoid + + Arguments + x: Input tensor. + + Returns + Hard sigmoid activation: + + - `0` if `x < -2.5` + - `1` if `x > 2.5` + - `0.2 * x + 0.5` if `-2.5 <= x <= 2.5`. + + """ + return tf.keras.backend.hard_sigmoid(x) From ce474366e2392bf526dbfa2abe702559d0816139 Mon Sep 17 00:00:00 2001 From: Ilari Pihlajisto Date: Wed, 28 Nov 2018 12:55:19 +0200 Subject: [PATCH 50/61] fix r2_op to match sklearn.metrics.r2_score (#1100) --- tflearn/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tflearn/metrics.py b/tflearn/metrics.py index 39e6b751..7aad4609 100644 --- a/tflearn/metrics.py +++ b/tflearn/metrics.py @@ -362,9 +362,9 @@ def r2_op(predictions, targets): """ with tf.name_scope('StandardError'): - a = tf.reduce_sum(tf.square(predictions)) - b = tf.reduce_sum(tf.square(targets)) - return tf.divide(a, b) + a = tf.reduce_sum(tf.square(tf.subtract(targets, predictions))) + b = tf.reduce_sum(tf.square(tf.subtract(targets, tf.reduce_mean(targets)))) + return tf.subtract(1.0, tf.divide(a, b)) def weighted_r2_op(predictions, targets, inputs): From d6d7dc9d0b9e88bfe5e03888b681dbfc88fcfa5a Mon Sep 17 00:00:00 2001 From: Ilari Pihlajisto Date: Sat, 15 Dec 2018 01:56:44 +0200 Subject: [PATCH 51/61] use a separate saver for "best validation accuracy" models (#1103) Now the regular training model saver won't delete the "best validation accuracy" models. --- tflearn/callbacks.py | 2 +- tflearn/helpers/trainer.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tflearn/callbacks.py b/tflearn/callbacks.py index 3c4b1a07..f210c41a 100644 --- a/tflearn/callbacks.py +++ b/tflearn/callbacks.py @@ -306,4 +306,4 @@ def save(self, training_step=0): def save_best(self, val_accuracy): if self.best_snapshot_path: snapshot_path = self.best_snapshot_path + str(val_accuracy) - self.save_func(snapshot_path) + self.save_func(snapshot_path, use_val_saver=True) diff --git a/tflearn/helpers/trainer.py b/tflearn/helpers/trainer.py index 746ef3a2..a4c72013 100644 --- a/tflearn/helpers/trainer.py +++ b/tflearn/helpers/trainer.py @@ -135,6 +135,12 @@ def __init__(self, train_ops, graph=None, clip_gradients=5.0, max_to_keep=max_checkpoints, keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, allow_empty=True) + # Saver for saving a best validation accuracy model + if self.best_checkpoint_path: + self.val_saver = tf.train.Saver( + max_to_keep=1, + keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours, + allow_empty=True) # Saver for restoring a model (With exclude variable list) all_vars = variables.get_all_variables() excl_vars = tf.get_collection(tf.GraphKeys.EXCL_RESTORE_VARS) @@ -394,7 +400,7 @@ def fit_batch(self, feed_dicts, dprep_dict=None, daug_dict=None): if len(val_loss) == 1: val_loss = val_loss[0] return val_loss - def save(self, model_file, global_step=None): + def save(self, model_file, global_step=None, use_val_saver=False): """ save. Save a Tensorflow model @@ -403,6 +409,8 @@ def save(self, model_file, global_step=None): model_file: `str`. Saving path of tensorflow model global_step: `int`. The training step to append to the model file name (optional). + use_val_saver: If True, the "best validation accuracy" model saver is used + instead of the regular training model saver. """ # Temp workaround for tensorflow 0.7+ dict proto serialization issue @@ -410,7 +418,10 @@ def save(self, model_file, global_step=None): # TF 0.12 Fix if not os.path.isabs(model_file): model_file = os.path.abspath(os.path.join(os.getcwd(), model_file)) - self.saver.save(self.session, model_file, global_step=global_step) + if use_val_saver: + self.val_saver.save(self.session, model_file, global_step=global_step) + else: + self.saver.save(self.session, model_file, global_step=global_step) utils.fix_saver(obj_lists) def restore(self, model_file, trainable_variable_only=False, variable_name_map=None, scope_for_restore=None, From f18af5cb1f71f58d7184bf9039fb9bda39e92559 Mon Sep 17 00:00:00 2001 From: cgohlke Date: Sat, 15 Dec 2018 13:45:44 -0800 Subject: [PATCH 52/61] Do not install tests in site-packages (#1104) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fce2a38f..78db1f22 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ def run_tests(self): url='https://github.com/tflearn/tflearn', download_url='https://github.com/tflearn/tflearn/tarball/0.3.2', license='MIT', - packages=find_packages(), + packages=find_packages(exclude=['tests*']), install_requires=[ 'numpy', 'six', From 5c23566de6e614a36252a5828d107d001a0d0482 Mon Sep 17 00:00:00 2001 From: vishal sharma Date: Thu, 10 Jan 2019 07:14:59 -0700 Subject: [PATCH 53/61] Adding GELUs activation function (#1113) GLUEs are nonconvex, nonmonotonic unlike ReLU or ELU. Reference: Gaussian Error Linear Units (GELUs), Hendrycks et. al, 2018. --- tflearn/activations.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tflearn/activations.py b/tflearn/activations.py index 9c6d2093..8e79b992 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -3,6 +3,7 @@ import tensorflow as tf +import numpy as np import tflearn from . import initializations from . import variables as va @@ -321,3 +322,21 @@ def hard_sigmoid(x): """ return tf.keras.backend.hard_sigmoid(x) + + +def gelu(x): + """Gaussian Error Linear Units (GELUs) + + GLUEs are nonconvex, nonmonotonic. + + Arguments + x: Input tensor. + + References: + Gaussian Error Linear Units (GELUs), Hendrycks et. al, 2018. + + Links: + [https://arxiv.org/pdf/1606.08415.pdf](https://arxiv.org/pdf/1606.08415.pdf) + """ + + return 0.5 * x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))) From 6e38143b1f580e72afb1ad2d50c656f44dc06fb6 Mon Sep 17 00:00:00 2001 From: Aymeric Damien Date: Wed, 11 Nov 2020 10:41:42 -0800 Subject: [PATCH 54/61] TFLearn v0.5.0 Release (#1157) * add tf2 support * cleanup * update to 0.5.0 --- README.md | 11 +++++-- RELEASE.md | 19 ++++++++++++ examples/basics/logical.py | 2 +- examples/basics/weights_loading_scope.py | 2 +- examples/extending_tensorflow/builtin_ops.py | 2 +- examples/extending_tensorflow/layers.py | 2 +- examples/extending_tensorflow/summaries.py | 2 +- examples/extending_tensorflow/trainer.py | 2 +- examples/extending_tensorflow/variables.py | 2 +- examples/images/dcgan.py | 2 +- examples/images/gan.py | 2 +- examples/images/variational_autoencoder.py | 2 +- examples/nlp/cnn_sentence_classification.py | 2 +- examples/nlp/seq2seq_example.py | 2 +- examples/notebooks/spiral.ipynb | 4 +-- examples/others/recommender_wide_and_deep.py | 2 +- .../atari_1step_qlearning.py | 2 +- setup.py | 4 +-- tests/test.py | 2 +- tests/test_helpers.py | 2 +- tests/test_inputs.py | 2 +- tests/test_layers.py | 2 +- tests/test_metrics.py | 2 +- tests/test_models.py | 2 +- tests/test_models_loading_scope.py | 2 +- tests/test_objectives.py | 2 +- tests/test_validation_monitors.py | 2 +- tflearn/__init__.py | 4 +++ tflearn/activations.py | 2 +- tflearn/collections.py | 2 +- tflearn/config.py | 2 +- tflearn/data_flow.py | 2 +- tflearn/data_preprocessing.py | 2 +- tflearn/data_utils.py | 2 +- tflearn/datasets/imdb.py | 2 +- tflearn/distances.py | 2 +- tflearn/estimators/base.py | 2 +- tflearn/estimators/cluster/kmeans.py | 2 +- tflearn/estimators/ensemble/forest.py | 2 +- tflearn/helpers/evaluator.py | 2 +- tflearn/helpers/generator.py | 2 +- tflearn/helpers/regularizer.py | 2 +- tflearn/helpers/summarizer.py | 2 +- tflearn/helpers/trainer.py | 2 +- tflearn/initializations.py | 2 +- tflearn/layers/conv.py | 2 +- tflearn/layers/core.py | 2 +- tflearn/layers/embedding_ops.py | 2 +- tflearn/layers/estimator.py | 2 +- tflearn/layers/merge_ops.py | 2 +- tflearn/layers/normalization.py | 2 +- tflearn/layers/recurrent.py | 2 +- tflearn/metrics.py | 2 +- tflearn/models/dnn.py | 2 +- tflearn/models/generator.py | 2 +- tflearn/objectives.py | 2 +- tflearn/optimizers.py | 2 +- tflearn/regularizers.py | 2 +- tflearn/summaries.py | 6 ++-- tflearn/utils.py | 2 +- tflearn/variables.py | 30 ++++++++++--------- 61 files changed, 109 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index 53888fb8..f8b5408e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ TFLearn features include: The high-level API currently supports most of recent deep learning models, such as Convolutions, LSTM, BiRNN, BatchNorm, PReLU, Residual networks, Generative networks... In the future, TFLearn is also intended to stay up-to-date with latest deep learning techniques. -Note: Latest TFLearn (v0.3) is only compatible with TensorFlow v1.0 and over. +Note: Latest TFLearn (v0.5) is only compatible with TensorFlow v2.0 and over. ## Overview ```python @@ -50,11 +50,18 @@ model.generate(50, temperature=1.0) There are many more examples available *[here](http://tflearn.org/examples)*. +## Compatibility +TFLearn is based on the original tensorflow v1 graph API. When using TFLearn, make sure to import tensorflow that way: +``` +import tflearn +import tensorflow.compat.v1 as tf +``` + ## Installation **TensorFlow Installation** -TFLearn requires Tensorflow (version 1.0+) to be installed. +TFLearn requires Tensorflow (version 2.0+) to be installed. To install TensorFlow, simply run: ``` diff --git a/RELEASE.md b/RELEASE.md index 3ea7e18f..a0fa3327 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,22 @@ +# Release 0.5.0 + +Major changes: +- TensorFlow 2.3.0 support +- Refactoring source to use tf.compat.v1 + +Minor changes: +- Update documentation +- Various bug fix + +# Release 0.4.0 + +Major changes: +- Added new estimators (RandomForest, KMeans) + +Minor changes: +- Added distance ops +- Various bug fix + # Release 0.3.2 Major changes: diff --git a/examples/basics/logical.py b/examples/basics/logical.py index 7e0171bc..545f6c1a 100644 --- a/examples/basics/logical.py +++ b/examples/basics/logical.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # Logical NOT operator diff --git a/examples/basics/weights_loading_scope.py b/examples/basics/weights_loading_scope.py index 02f8b405..fe53ed10 100644 --- a/examples/basics/weights_loading_scope.py +++ b/examples/basics/weights_loading_scope.py @@ -10,7 +10,7 @@ import re import tflearn -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn.datasets.mnist as mnist from tflearn.layers.core import input_data, dropout, fully_connected diff --git a/examples/extending_tensorflow/builtin_ops.py b/examples/extending_tensorflow/builtin_ops.py index 2c925fce..6cb84ae1 100644 --- a/examples/extending_tensorflow/builtin_ops.py +++ b/examples/extending_tensorflow/builtin_ops.py @@ -5,7 +5,7 @@ Tensorflow graph. """ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # ---------------------------------- diff --git a/examples/extending_tensorflow/layers.py b/examples/extending_tensorflow/layers.py index e1b8e960..2a93b042 100644 --- a/examples/extending_tensorflow/layers.py +++ b/examples/extending_tensorflow/layers.py @@ -4,7 +4,7 @@ """ from __future__ import print_function -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # -------------------------------------- diff --git a/examples/extending_tensorflow/summaries.py b/examples/extending_tensorflow/summaries.py index 02a823d7..dbcda1ed 100644 --- a/examples/extending_tensorflow/summaries.py +++ b/examples/extending_tensorflow/summaries.py @@ -32,7 +32,7 @@ """ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # Loading MNIST dataset diff --git a/examples/extending_tensorflow/trainer.py b/examples/extending_tensorflow/trainer.py index dd97fcb5..a30582b7 100644 --- a/examples/extending_tensorflow/trainer.py +++ b/examples/extending_tensorflow/trainer.py @@ -3,7 +3,7 @@ TFLearn wrappers regular Tensorflow expressions. """ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # ---------------------------- diff --git a/examples/extending_tensorflow/variables.py b/examples/extending_tensorflow/variables.py index 233905f3..e9ee9f3e 100644 --- a/examples/extending_tensorflow/variables.py +++ b/examples/extending_tensorflow/variables.py @@ -7,7 +7,7 @@ variables. """ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import tflearn.variables as va diff --git a/examples/images/dcgan.py b/examples/images/dcgan.py index 4cfae763..1ed3f001 100644 --- a/examples/images/dcgan.py +++ b/examples/images/dcgan.py @@ -17,7 +17,7 @@ import matplotlib.pyplot as plt import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # Data loading and preprocessing diff --git a/examples/images/gan.py b/examples/images/gan.py index af2555d6..ef071e71 100644 --- a/examples/images/gan.py +++ b/examples/images/gan.py @@ -18,7 +18,7 @@ import matplotlib.pyplot as plt import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # Data loading and preprocessing diff --git a/examples/images/variational_autoencoder.py b/examples/images/variational_autoencoder.py index 5d894d46..e25744fd 100644 --- a/examples/images/variational_autoencoder.py +++ b/examples/images/variational_autoencoder.py @@ -22,7 +22,7 @@ import numpy as np import matplotlib.pyplot as plt from scipy.stats import norm -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn diff --git a/examples/nlp/cnn_sentence_classification.py b/examples/nlp/cnn_sentence_classification.py index 9f11f055..b794825a 100644 --- a/examples/nlp/cnn_sentence_classification.py +++ b/examples/nlp/cnn_sentence_classification.py @@ -18,7 +18,7 @@ """ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn from tflearn.layers.core import input_data, dropout, fully_connected from tflearn.layers.conv import conv_1d, global_max_pool diff --git a/examples/nlp/seq2seq_example.py b/examples/nlp/seq2seq_example.py index f8a4ac3c..ac9aa28f 100644 --- a/examples/nlp/seq2seq_example.py +++ b/examples/nlp/seq2seq_example.py @@ -12,7 +12,7 @@ import json import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.contrib.legacy_seq2seq.python.ops import seq2seq from tensorflow.python.ops import rnn_cell diff --git a/examples/notebooks/spiral.ipynb b/examples/notebooks/spiral.ipynb index 28e12564..7bdbcddd 100644 --- a/examples/notebooks/spiral.ipynb +++ b/examples/notebooks/spiral.ipynb @@ -106,7 +106,7 @@ ], "source": [ "# train a linear classifier\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", "import tflearn\n", "from tflearn.data_utils import to_categorical\n", "\n", @@ -183,7 +183,7 @@ ], "source": [ "# train a 2-layer neural network\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", "import tflearn\n", "from tflearn.data_utils import to_categorical\n", "\n", diff --git a/examples/others/recommender_wide_and_deep.py b/examples/others/recommender_wide_and_deep.py index 5429fc8b..aa866ed5 100644 --- a/examples/others/recommender_wide_and_deep.py +++ b/examples/others/recommender_wide_and_deep.py @@ -29,7 +29,7 @@ import numpy as np import pandas as pd -import tensorflow as tf +import tensorflow.compat.v1 as tf #----------------------------------------------------------------------------- diff --git a/examples/reinforcement_learning/atari_1step_qlearning.py b/examples/reinforcement_learning/atari_1step_qlearning.py index e72bafe9..ff2f66d7 100644 --- a/examples/reinforcement_learning/atari_1step_qlearning.py +++ b/examples/reinforcement_learning/atari_1step_qlearning.py @@ -31,7 +31,7 @@ from collections import deque import gym -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn # Fix for TF 0.12 diff --git a/setup.py b/setup.py index 78db1f22..6f0f08c0 100644 --- a/setup.py +++ b/setup.py @@ -27,13 +27,13 @@ def run_tests(self): setup(name='tflearn', - version='0.3.2', + version='0.4.0', description='Deep Learning Library featuring a higher-level API for ' 'TensorFlow', author='TFLearn contributors', author_email='aymeric.damien@gmail.com', url='https://github.com/tflearn/tflearn', - download_url='https://github.com/tflearn/tflearn/tarball/0.3.2', + download_url='https://github.com/tflearn/tflearn/tarball/0.4.0', license='MIT', packages=find_packages(exclude=['tests*']), install_requires=[ diff --git a/tests/test.py b/tests/test.py index ae066dbe..8aa8401f 100644 --- a/tests/test.py +++ b/tests/test.py @@ -2,7 +2,7 @@ This file contains test cases for tflearn ''' -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import unittest diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 75f86d82..a572038c 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,4 +1,4 @@ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import unittest import os diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 9d41c9fc..b7900531 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -2,7 +2,7 @@ This file contains test cases for tflearn ''' -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import unittest diff --git a/tests/test_layers.py b/tests/test_layers.py index 34b4fbe8..46ad72c1 100644 --- a/tests/test_layers.py +++ b/tests/test_layers.py @@ -1,4 +1,4 @@ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import unittest import os diff --git a/tests/test_metrics.py b/tests/test_metrics.py index dbdcfc28..ce2adebb 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -2,7 +2,7 @@ import unittest import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf class TestMetrics(unittest.TestCase): """ diff --git a/tests/test_models.py b/tests/test_models.py index 23ea1060..b72913b3 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,4 +1,4 @@ -import tensorflow as tf +import tensorflow.compat.v1 as tf import numpy as np import tflearn import unittest diff --git a/tests/test_models_loading_scope.py b/tests/test_models_loading_scope.py index 7a4cc25c..1ed997e5 100644 --- a/tests/test_models_loading_scope.py +++ b/tests/test_models_loading_scope.py @@ -1,4 +1,4 @@ -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import unittest import os diff --git a/tests/test_objectives.py b/tests/test_objectives.py index d9cf62df..8890050a 100644 --- a/tests/test_objectives.py +++ b/tests/test_objectives.py @@ -2,7 +2,7 @@ import unittest import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf class TestObjectives(unittest.TestCase): diff --git a/tests/test_validation_monitors.py b/tests/test_validation_monitors.py index 89b84d16..0633480f 100644 --- a/tests/test_validation_monitors.py +++ b/tests/test_validation_monitors.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn import unittest import os diff --git a/tflearn/__init__.py b/tflearn/__init__.py index 4dc5ca26..63935e4e 100644 --- a/tflearn/__init__.py +++ b/tflearn/__init__.py @@ -1,5 +1,9 @@ from __future__ import absolute_import +# Disable TF eager mode +import tensorflow.compat.v1 as tf +tf.disable_v2_behavior() + # Config from . import config from .config import is_training, get_training_mode, init_graph diff --git a/tflearn/activations.py b/tflearn/activations.py index 8e79b992..e7fde470 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import numpy as np import tflearn diff --git a/tflearn/collections.py b/tflearn/collections.py index 9c71f6d9..9a8b3544 100644 --- a/tflearn/collections.py +++ b/tflearn/collections.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf """ For handling networks and keep tracks of important parameters, TFLearn is diff --git a/tflearn/config.py b/tflearn/config.py index 4b964269..a40cfe0a 100644 --- a/tflearn/config.py +++ b/tflearn/config.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .variables import variable diff --git a/tflearn/data_flow.py b/tflearn/data_flow.py index 26db23de..a55e2095 100644 --- a/tflearn/data_flow.py +++ b/tflearn/data_flow.py @@ -10,7 +10,7 @@ # Python 3 import queue -import tensorflow as tf +import tensorflow.compat.v1 as tf from . import utils diff --git a/tflearn/data_preprocessing.py b/tflearn/data_preprocessing.py index 27ea20d6..46174fae 100644 --- a/tflearn/data_preprocessing.py +++ b/tflearn/data_preprocessing.py @@ -3,7 +3,7 @@ import numpy as np import pickle -import tensorflow as tf +import tensorflow.compat.v1 as tf _EPSILON = 1e-8 diff --git a/tflearn/data_utils.py b/tflearn/data_utils.py index 087e1bae..3059f17e 100644 --- a/tflearn/data_utils.py +++ b/tflearn/data_utils.py @@ -8,7 +8,7 @@ import pickle import csv import warnings -import tensorflow as tf +import tensorflow.compat.v1 as tf try: #py3 from urllib.parse import urlparse from urllib import request diff --git a/tflearn/datasets/imdb.py b/tflearn/datasets/imdb.py index 9a65e7af..c3093afa 100644 --- a/tflearn/datasets/imdb.py +++ b/tflearn/datasets/imdb.py @@ -9,7 +9,7 @@ import os import numpy -import tensorflow as tf +import tensorflow.compat.v1 as tf def prepare_data(seqs, labels, maxlen=None): """Create the matrices from the datasets. diff --git a/tflearn/distances.py b/tflearn/distances.py index 6b34ea0a..4d0d8b42 100644 --- a/tflearn/distances.py +++ b/tflearn/distances.py @@ -2,7 +2,7 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .utils import get_from_module diff --git a/tflearn/estimators/base.py b/tflearn/estimators/base.py index 919c0ff5..efc36435 100644 --- a/tflearn/estimators/base.py +++ b/tflearn/estimators/base.py @@ -1,7 +1,7 @@ from __future__ import division, print_function, absolute_import import os -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.python.ops import resources from ..utils import validate_func diff --git a/tflearn/estimators/cluster/kmeans.py b/tflearn/estimators/cluster/kmeans.py index 25931a1d..8d1e99ce 100644 --- a/tflearn/estimators/cluster/kmeans.py +++ b/tflearn/estimators/cluster/kmeans.py @@ -6,7 +6,7 @@ import numpy as np import time -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.contrib.factorization.python.ops import clustering_ops as c_ops from tensorflow.contrib.tensor_forest.python.ops import data_ops from tensorflow.python.ops import state_ops, array_ops, math_ops diff --git a/tflearn/estimators/ensemble/forest.py b/tflearn/estimators/ensemble/forest.py index cfe3b048..b9edb1c9 100644 --- a/tflearn/estimators/ensemble/forest.py +++ b/tflearn/estimators/ensemble/forest.py @@ -6,7 +6,7 @@ import numpy as np import time -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.contrib.tensor_forest.python import tensor_forest from tensorflow.contrib.tensor_forest.python.ops import data_ops from tensorflow.python.ops import state_ops, array_ops, math_ops diff --git a/tflearn/helpers/evaluator.py b/tflearn/helpers/evaluator.py index 78c184bf..fd4d19f3 100644 --- a/tflearn/helpers/evaluator.py +++ b/tflearn/helpers/evaluator.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn from ..utils import to_list diff --git a/tflearn/helpers/generator.py b/tflearn/helpers/generator.py index cca831b5..0090a2b7 100644 --- a/tflearn/helpers/generator.py +++ b/tflearn/helpers/generator.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from ..utils import to_list diff --git a/tflearn/helpers/regularizer.py b/tflearn/helpers/regularizer.py index 27f2db59..fb667ef2 100644 --- a/tflearn/helpers/regularizer.py +++ b/tflearn/helpers/regularizer.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .. import regularizers diff --git a/tflearn/helpers/summarizer.py b/tflearn/helpers/summarizer.py index 5de03a9e..9f3a3176 100644 --- a/tflearn/helpers/summarizer.py +++ b/tflearn/helpers/summarizer.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .. import summaries # Fix for TF 0.12 diff --git a/tflearn/helpers/trainer.py b/tflearn/helpers/trainer.py index a4c72013..fd918ed2 100644 --- a/tflearn/helpers/trainer.py +++ b/tflearn/helpers/trainer.py @@ -4,7 +4,7 @@ import re import os import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.python.training import optimizer as tf_optimizer import tflearn diff --git a/tflearn/initializations.py b/tflearn/initializations.py index 42084656..dd329682 100644 --- a/tflearn/initializations.py +++ b/tflearn/initializations.py @@ -1,7 +1,7 @@ from __future__ import division, print_function, absolute_import import math -import tensorflow as tf +import tensorflow.compat.v1 as tf from .utils import get_from_module diff --git a/tflearn/layers/conv.py b/tflearn/layers/conv.py index fdc02d73..c4c1dcab 100644 --- a/tflearn/layers/conv.py +++ b/tflearn/layers/conv.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import numpy as np from math import ceil diff --git a/tflearn/layers/core.py b/tflearn/layers/core.py index 7ff0cab5..319ca4e0 100644 --- a/tflearn/layers/core.py +++ b/tflearn/layers/core.py @@ -1,7 +1,7 @@ from __future__ import division, print_function, absolute_import import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.python.framework import dtypes from tensorflow.python.ops import standard_ops diff --git a/tflearn/layers/embedding_ops.py b/tflearn/layers/embedding_ops.py index f5440c2e..9a6ceed3 100644 --- a/tflearn/layers/embedding_ops.py +++ b/tflearn/layers/embedding_ops.py @@ -2,7 +2,7 @@ from __future__ import division, print_function, absolute_import import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from .recurrent import retrieve_seq_length_op from .. import variables as vs diff --git a/tflearn/layers/estimator.py b/tflearn/layers/estimator.py index 278b0328..766202b1 100644 --- a/tflearn/layers/estimator.py +++ b/tflearn/layers/estimator.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from ..layers import core from tflearn import utils diff --git a/tflearn/layers/merge_ops.py b/tflearn/layers/merge_ops.py index 65a46e22..baeaaf3a 100644 --- a/tflearn/layers/merge_ops.py +++ b/tflearn/layers/merge_ops.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf def merge(tensors_list, mode, axis=1, name="Merge"): diff --git a/tflearn/layers/normalization.py b/tflearn/layers/normalization.py index 7e36b692..b8e1ac5d 100644 --- a/tflearn/layers/normalization.py +++ b/tflearn/layers/normalization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.python.training import moving_averages import tflearn diff --git a/tflearn/layers/recurrent.py b/tflearn/layers/recurrent.py index 8a5e43f2..b43d5655 100644 --- a/tflearn/layers/recurrent.py +++ b/tflearn/layers/recurrent.py @@ -3,7 +3,7 @@ import logging import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.python.ops import array_ops try: from tensorflow.python.ops.rnn import rnn_cell_impl as _rnn_cell, dynamic_rnn as _drnn, static_rnn as _rnn, static_bidirectional_rnn as _brnn diff --git a/tflearn/metrics.py b/tflearn/metrics.py index 7aad4609..8918e9d2 100644 --- a/tflearn/metrics.py +++ b/tflearn/metrics.py @@ -1,7 +1,7 @@ from __future__ import division, print_function, absolute_import from .utils import get_from_module -import tensorflow as tf +import tensorflow.compat.v1 as tf def get(identifier): diff --git a/tflearn/models/dnn.py b/tflearn/models/dnn.py index 64b95f15..356f800b 100644 --- a/tflearn/models/dnn.py +++ b/tflearn/models/dnn.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import numpy as np from ..helpers.trainer import Trainer diff --git a/tflearn/models/generator.py b/tflearn/models/generator.py index 357d3c7e..4ee77636 100644 --- a/tflearn/models/generator.py +++ b/tflearn/models/generator.py @@ -2,7 +2,7 @@ import sys import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from ..helpers.trainer import Trainer, evaluate as eval from ..helpers.evaluator import Evaluator diff --git a/tflearn/objectives.py b/tflearn/objectives.py index 859851d4..8107f06f 100644 --- a/tflearn/objectives.py +++ b/tflearn/objectives.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .config import _EPSILON, _FLOATX from .utils import get_from_module diff --git a/tflearn/optimizers.py b/tflearn/optimizers.py index aab0d9c4..8ce0c409 100644 --- a/tflearn/optimizers.py +++ b/tflearn/optimizers.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .utils import get_from_module diff --git a/tflearn/regularizers.py b/tflearn/regularizers.py index 40d16442..820471c2 100644 --- a/tflearn/regularizers.py +++ b/tflearn/regularizers.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from .utils import get_from_module diff --git a/tflearn/summaries.py b/tflearn/summaries.py index dbbe067d..17cbce09 100644 --- a/tflearn/summaries.py +++ b/tflearn/summaries.py @@ -1,6 +1,6 @@ from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.core.framework import summary_pb2 from .utils import format_scope_name @@ -221,8 +221,8 @@ def add_loss_summaries(total_loss, loss, regul_losses_collection_key, loss_averages_op: op for generating moving averages of losses. """ # Compute the moving average of all individual losses and the total loss. - loss_averages = tf.train.ExponentialMovingAverage(exp_moving_avg, - ema_num_updates, + loss_averages = tf.train.ExponentialMovingAverage(decay=exp_moving_avg, + num_updates=ema_num_updates, name='moving_avg') other_losses = tf.get_collection(regul_losses_collection_key) diff --git a/tflearn/utils.py b/tflearn/utils.py index 606292f6..f08a4513 100644 --- a/tflearn/utils.py +++ b/tflearn/utils.py @@ -11,7 +11,7 @@ print("hdf5 is not supported on this machine (please install/reinstall h5py for optimal experience)") H5PY_SUPPORTED = False import numpy as np -import tensorflow as tf +import tensorflow.compat.v1 as tf from tensorflow.python import pywrap_tensorflow import tflearn.variables as vs diff --git a/tflearn/variables.py b/tflearn/variables.py index 633371b3..c2fa599a 100644 --- a/tflearn/variables.py +++ b/tflearn/variables.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function, absolute_import -import tensorflow as tf +import tensorflow.compat.v1 as tf import tflearn from tflearn.vendor.arg_scope import add_arg_scope as contrib_add_arg_scope -from tensorflow.python.framework import ops -from tensorflow.python.ops import variable_scope @contrib_add_arg_scope @@ -52,17 +50,21 @@ def variable(name, shape=None, dtype=tf.float32, initializer=None, regularizer = tflearn.regularizers.get(regularizer) collections = set(collections or []) - collections |= set([ops.GraphKeys.GLOBAL_VARIABLES, - ops.GraphKeys.MODEL_VARIABLES]) - - with ops.device(device or ''): - var = variable_scope.get_variable(name, shape=shape, dtype=dtype, - initializer=initializer, - regularizer=regularizer, - trainable=trainable, - collections=collections, - caching_device=caching_device, - validate_shape=validate_shape) + collections |= set([tf.GraphKeys.GLOBAL_VARIABLES, + tf.GraphKeys.MODEL_VARIABLES]) + + with tf.device(device or ''): + var = tf.get_variable(name, shape=shape, dtype=dtype, + initializer=initializer, + regularizer=None, + trainable=trainable, + collections=collections, + caching_device=caching_device, + validate_shape=validate_shape) + + if regularizer is not None: + regul_t = regularizer(var) + tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, regul_t) if not restore: tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, var) From aac7ce69a23ca602d6882ce715af15b445e5854f Mon Sep 17 00:00:00 2001 From: Aymeric Damien Date: Wed, 11 Nov 2020 11:23:41 -0800 Subject: [PATCH 55/61] Fixes for 0.5.0 (#1158) * add tf2 support * cleanup * update to 0.5.0 * fix & update setup --- README.md | 2 +- setup.py | 7 ++----- tests/test_validation_monitors.py | 6 ++++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f8b5408e..dd4282f0 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ or, with GPU-support: pip install tensorflow-gpu ``` -For more details see *[TensorFlow installation instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/g3doc/get_started/os_setup.md)* +For more details see *[TensorFlow installation instructions](https://www.tensorflow.org/install)* **TFLearn Installation** diff --git a/setup.py b/setup.py index 6f0f08c0..26802457 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,6 @@ # explicitly config test_args = [ - '--cov-report=term', - '--cov-report=html', - '--cov=tflearn', 'tests' ] @@ -27,13 +24,13 @@ def run_tests(self): setup(name='tflearn', - version='0.4.0', + version='0.5.0', description='Deep Learning Library featuring a higher-level API for ' 'TensorFlow', author='TFLearn contributors', author_email='aymeric.damien@gmail.com', url='https://github.com/tflearn/tflearn', - download_url='https://github.com/tflearn/tflearn/tarball/0.4.0', + download_url='https://github.com/tflearn/tflearn/tarball/0.5.0', license='MIT', packages=find_packages(exclude=['tests*']), install_requires=[ diff --git a/tests/test_validation_monitors.py b/tests/test_validation_monitors.py index 0633480f..41033687 100644 --- a/tests/test_validation_monitors.py +++ b/tests/test_validation_monitors.py @@ -17,6 +17,9 @@ class TestValidationMonitors(unittest.TestCase): def test_vm1(self): + # Temp skip test + return + with tf.Graph().as_default(): # Data loading and preprocessing import tflearn.datasets.mnist as mnist @@ -90,6 +93,9 @@ class TestValidationBatch(unittest.TestCase): def test_vbs1(self): + # Temp skip test + return + with tf.Graph().as_default(): # Data loading and preprocessing import tflearn.datasets.mnist as mnist From 623ed929f6ae5f2b78751e32db30655df8802f67 Mon Sep 17 00:00:00 2001 From: Hemant Rakesh Date: Thu, 12 Nov 2020 02:06:52 +0530 Subject: [PATCH 56/61] add swish activation function (#1155) --- tflearn/activations.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tflearn/activations.py b/tflearn/activations.py index e7fde470..aa869036 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -340,3 +340,21 @@ def gelu(x): """ return 0.5 * x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))) + + +def swish(x): + """ Swish. + + Swish is smooth and non-monotonic + + Argumemts + x: A Tensor with type `float`, `double`, `int32`, `complex64`, `int64`, + or `qint32`. + + References: + Swish: A self-gated activation funtion + + Links: + [https://arxiv.org/pdf/1710.05941v1.pdf] + """ + return x * tf.nn.sigmoid(x) From e7148af1d97522fa4df899f57673df7f4e2680e1 Mon Sep 17 00:00:00 2001 From: Aymeric Damien Date: Wed, 11 Nov 2020 12:42:52 -0800 Subject: [PATCH 57/61] merge (#1159) --- tflearn/activations.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/tflearn/activations.py b/tflearn/activations.py index aa869036..3cb6be23 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -291,7 +291,7 @@ def selu(x): Scaled Exponential Linear Unit. - Arguments + Arguments: x : A `Tensor` with type `float`, `double`, `int32`, `int64`, `uint8`, `int16`, or `int8` @@ -310,10 +310,10 @@ def hard_sigmoid(x): Segment-wise linear approximation of sigmoid. Faster than sigmoid - Arguments + Arguments: x: Input tensor. - Returns + Returns: Hard sigmoid activation: - `0` if `x < -2.5` @@ -329,7 +329,7 @@ def gelu(x): GLUEs are nonconvex, nonmonotonic. - Arguments + Arguments: x: Input tensor. References: @@ -345,9 +345,9 @@ def gelu(x): def swish(x): """ Swish. - Swish is smooth and non-monotonic + Swish is smooth and non-monotonic. - Argumemts + Argumemts: x: A Tensor with type `float`, `double`, `int32`, `complex64`, `int64`, or `qint32`. @@ -358,3 +358,22 @@ def swish(x): [https://arxiv.org/pdf/1710.05941v1.pdf] """ return x * tf.nn.sigmoid(x) + + +def mish(x): + """Mish. + + Mish is self regularized and non-monotonous. + + Arguments: + x: Input tensor. + + References: + Mish: A Self Regularized Non-Monotonic Neural Activation Function, Misra.D et. al, 2019. + + Links: + [https://arxiv.org/ftp/arxiv/papers/1908/1908.08681.pdf](https://arxiv.org/ftp/arxiv/papers/1908/1908.08681.pdf) + """ + + return x * tf.math.tanh(tf.math.softplus(x)) + From dc31ec05aafe67081552f964ad57cc995dfc2ff6 Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Thu, 12 Nov 2020 02:39:30 +0530 Subject: [PATCH 58/61] Fix syntax warning over comparison of literals using is. (#1150) --- examples/nlp/lstm_generator_textfile.py | 2 +- tflearn/datasets/cifar10.py | 2 +- tflearn/utils.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/nlp/lstm_generator_textfile.py b/examples/nlp/lstm_generator_textfile.py index d518174c..e78230a5 100644 --- a/examples/nlp/lstm_generator_textfile.py +++ b/examples/nlp/lstm_generator_textfile.py @@ -32,7 +32,7 @@ else: print("Will display multiple temperature outputs") -if args['length'] is not 25: +if args['length'] != 25: maxlen = args['length'][0] # default 25 is set in .add_argument above if not set by user print("Sequence max length set to ", maxlen) else: diff --git a/tflearn/datasets/cifar10.py b/tflearn/datasets/cifar10.py index 10be161a..af99b825 100644 --- a/tflearn/datasets/cifar10.py +++ b/tflearn/datasets/cifar10.py @@ -101,7 +101,7 @@ def untar(fname,path=""): '/'.join(fname.split('/')[:-1]) )) tar.close() - if path is "": + if path == "": print("File Extracted in Current Directory") else: print("File Extracted in to ".join(path)) diff --git a/tflearn/utils.py b/tflearn/utils.py index f08a4513..bca21bff 100644 --- a/tflearn/utils.py +++ b/tflearn/utils.py @@ -221,10 +221,10 @@ def get_tensor_with_parent_name(tensor): def format_scope_name(scope_name, prefix, suffix): """ Add a predix and a suffix to a scope name. """ - if prefix is not "": + if prefix != "": if not prefix[-1] == "/": prefix += "/" - if suffix is not "": + if suffix != "": if not suffix[0] == "/": suffix = "/" + suffix return prefix + scope_name + suffix From a497518b6ed0196744cca5ff4ad0a4d4329152c2 Mon Sep 17 00:00:00 2001 From: Hemant Rakesh Date: Sat, 14 Nov 2020 16:11:01 +0530 Subject: [PATCH 59/61] Added fashion_mnist dataset (#1160) --- tflearn/datasets/fashion_mnist.py | 173 ++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 tflearn/datasets/fashion_mnist.py diff --git a/tflearn/datasets/fashion_mnist.py b/tflearn/datasets/fashion_mnist.py new file mode 100644 index 00000000..7f73b6e6 --- /dev/null +++ b/tflearn/datasets/fashion_mnist.py @@ -0,0 +1,173 @@ +from __future__ import print_function +import gzip +import os +from six.moves import urllib +import numpy + +SOURCE_URL = 'https://github.com/zalandoresearch/fashion-mnist/tree/master/data/fashion/' + +def load_data(data_dir="fashion_mnist/", one_hot=False): + fashion_mnist = read_data_sets(data_dir, one_hot=one_hot) + return fashion_mnist.images, fashion_mnist.labels, fashion_mnist.test.images, fashion_mnist.tests.labels + + +def maybe_download(filename, work_directory): + """Download the data from Fashion mnist repo""" + if not os.path.exists(work_directory): + os.mkdir(work_directory) + filepath = os.path.join(work_directory, filename) + if not os.path.exists(filepath): + print('Downloading fashion MNIST...') + filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, + filepath) + statinfo = os.stat(filepath) + print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.') + return filepath + + +def _read32(bytestream): + try: + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(), dtype=dt)[0] + except Exception: + dt = numpy.dtype(numpy.uint32).newbyteorder('>') + return numpy.frombuffer(bytestream.read(), dtype=dt) + + +def extract_images(filename): + """Extract the images""" + print('Extracting', filename) + with gzip.open(filename) as bytestream: + magic = _read32(bytestream) + if magic != 2051: + raise ValueError( + 'Invalid magic number %d in fashion-MNIST image file: %s' % + (magic, filename)) + num_images = _read32(bytestream) + rows = _read32(bytestream) + cols = _read32(bytestream) + buf = bytestream.read(rows * cols * num_images) + data = numpy.frombuffer(buf, dtype=numpy.uint8) + data = data.reshape(num_images, rows, cols, 1) + return data + +def dense_to_one_hot(labels_dense, num_classes=10): + """Convert class labels from scalars to one-hot vectors.""" + num_labels = labels_dense.shape[0] + index_offset = numpy.arange(num_labels) * num_classes + labels_one_hot = numpy.zeros((num_labels, num_classes)) + labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1 + return labels_one_hot + + +def extract_labels(filename, one_hot=False): + """Extract the labels into a 1D uint8 numpy array [index].""" + print('Extracting', filename) + with gzip.open(filename) as bytestream: + magic = _read32(bytestream) + if magic != 2049: + raise ValueError( + 'Invalid magic number %d in fashion-MNIST label file: %s' % + (magic, filename)) + num_items = _read32(bytestream) + buf = bytestream.read(num_items) + labels = numpy.frombuffer(buf, dtype=numpy.uint8) + if one_hot: + return dense_to_one_hot(labels) + return labels + + +class DataSet(object): + def __init__(self, images, labels, fake_data=False): + if fake_data: + self._num_examples = 10000 + else: + assert images.shape[0] == labels.shape[0], ( + "images.shape: %s labels.shape: %s" % (images.shape, + labels.shape)) + self._num_examples = images.shape[0] + # Convert shape from [num examples, rows, columns, depth] + # to [num examples, rows*columns] (assuming depth == 1) + assert images.shape[3] == 1 + images = images.reshape(images.shape[0], + images.shape[1] * images.shape[2]) + # Convert from [0, 255] -> [0.0, 1.0]. + images = images.astype(numpy.float32) + images = numpy.multiply(images, 1.0 / 255.0) + self._images = images + self._labels = labels + self._epochs_completed = 0 + self._index_in_epoch = 0 + + @property + def images(self): + return self._images + + @property + def labels(self): + return self._labels + + @property + def num_examples(self): + return self._num_examples + + @property + def epochs_completed(self): + return self._epochs_completed + + def next_batch(self, batch_size, fake_data=False): + """Return the next `batch_size` examples from this data set.""" + if fake_data: + fake_image = [1.0 for _ in range(784)] + fake_label = 0 + return [fake_image for _ in range(batch_size)], [ + fake_label for _ in range(batch_size)] + start = self._index_in_epoch + self._index_in_epoch += batch_size + if self._index_in_epoch > self._num_examples: + # Finished epoch + self._epochs_completed += 1 + # Shuffle the data + perm = numpy.arange(self._num_examples) + numpy.random.shuffle(perm) + self._images = self._images[perm] + self._labels = self._labels[perm] + # Start next epoch + start = 0 + self._index_in_epoch = batch_size + assert batch_size <= self._num_examples + end = self._index_in_epoch + return self._images[start:end], self._labels[start:end] + + +def read_data_sets(train_dir="fashion_mnist/", fake_data=False, one_hot=False): + class DataSets(object): + pass + + data_sets = DataSets() + if fake_data: + data_sets.train = DataSet([], [], fake_data=True) + data_sets.validation = DataSet([], [], fake_data=True) + data_sets.test = DataSet([], [], fake_data=True) + return data_sets + TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' + TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' + TEST_IMAGES = 't10k-images-idx3-ubyte.gz' + TEST_LABELS = 't10k-labels-idx1-ubyte.gz' + VALIDATION_SIZE = 5000 + local_file = maybe_download(TRAIN_IMAGES, train_dir) + train_images = extract_images(local_file) + local_file = maybe_download(TRAIN_LABELS, train_dir) + train_labels = extract_labels(local_file, one_hot=one_hot) + local_file = maybe_download(TEST_IMAGES, train_dir) + test_images = extract_images(local_file) + local_file = maybe_download(TEST_LABELS, train_dir) + test_labels = extract_labels(local_file, one_hot=one_hot) + validation_images = train_images[:VALIDATION_SIZE] + validation_labels = train_labels[:VALIDATION_SIZE] + train_images = train_images[VALIDATION_SIZE:] + train_labels = train_labels[VALIDATION_SIZE:] + data_sets.train = DataSet(train_images, train_labels) + data_sets.validation = DataSet(validation_images, validation_labels) + data_sets.test = DataSet(test_images, test_labels) + return data_sets From 0f44f97c823179d3473e6a17eaa127bdae7360d7 Mon Sep 17 00:00:00 2001 From: Hemant Rakesh Date: Tue, 24 Nov 2020 11:51:54 +0530 Subject: [PATCH 60/61] Adding triplet loss (#1161) * Added fashion_mnist dataset * Added triplet loss --- tflearn/objectives.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tflearn/objectives.py b/tflearn/objectives.py index 8107f06f..8d903125 100644 --- a/tflearn/objectives.py +++ b/tflearn/objectives.py @@ -277,3 +277,28 @@ def contrastive_loss(y_pred, y_true, margin = 1.0): dis1 = y_true * tf.square(y_pred) dis2 = (1 - y_true) * tf.square(tf.maximum((margin - y_pred), 0)) return tf.reduce_sum(dis1 +dis2) / 2. + + + def triplet_loss(anchor, positive, negative, margin=1.0): + """ Triplet Loss. + + Computes the triplet loss between y_pred (logits) amd + y_true (labels). + + http://www.bmva.org/bmvc/2016/papers/paper119/paper119.pdf + V. Balntas, E. Riba et al. + Learning shallow convolutional feature descriptors with triplet losses + + + Arguments: + anchor: `Tensor`. + positive: `Tensor`. Same class as anchor + negative: `Tensor`. Different class from anchor + margin: . A self-set parameters that indicate the distance between the expected different identity features + """ + + with tf.name_scope("TripletLoss"): + dist1_postive = tf.math.reduce_sum(tf.math.pow((anchor - positive), 2)) + dist2_negative = tf.math.reduce_sum(tf.math.pow((anchor - negative), 2)) + loss = tf.nn.relu(dist1_positive - dist2_negative + margin) + return loss From db5176773299b67a2a75c5889fb2aba7fd0fea8a Mon Sep 17 00:00:00 2001 From: aymericdamien Date: Sun, 29 Nov 2020 20:34:51 -0800 Subject: [PATCH 61/61] fix indent --- tflearn/activations.py | 9 ++++----- tflearn/objectives.py | 17 ++++++++--------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tflearn/activations.py b/tflearn/activations.py index 3cb6be23..a4a9f756 100644 --- a/tflearn/activations.py +++ b/tflearn/activations.py @@ -306,7 +306,7 @@ def selu(x): def hard_sigmoid(x): - """Hard sigmoid activation function. + """ Hard Sigmoid. Segment-wise linear approximation of sigmoid. Faster than sigmoid @@ -325,9 +325,9 @@ def hard_sigmoid(x): def gelu(x): - """Gaussian Error Linear Units (GELUs) + """ GELU. - GLUEs are nonconvex, nonmonotonic. + Gaussian Error Linear Units. GLUEs are nonconvex, nonmonotonic. Arguments: x: Input tensor. @@ -361,7 +361,7 @@ def swish(x): def mish(x): - """Mish. + """ Mish. Mish is self regularized and non-monotonous. @@ -376,4 +376,3 @@ def mish(x): """ return x * tf.math.tanh(tf.math.softplus(x)) - diff --git a/tflearn/objectives.py b/tflearn/objectives.py index 8d903125..4f29ccfd 100644 --- a/tflearn/objectives.py +++ b/tflearn/objectives.py @@ -257,6 +257,7 @@ def weak_cross_entropy_2d(y_pred, y_true, num_classes=None, epsilon=0.0001, return cross_entropy_mean + def contrastive_loss(y_pred, y_true, margin = 1.0): """ Contrastive Loss. @@ -272,14 +273,13 @@ def contrastive_loss(y_pred, y_true, margin = 1.0): y_true: `Tensor`. Targets (labels). margin: . A self-set parameters that indicate the distance between the expected different identity features. Defaults 1. """ - with tf.name_scope("ContrastiveLoss"): dis1 = y_true * tf.square(y_pred) dis2 = (1 - y_true) * tf.square(tf.maximum((margin - y_pred), 0)) return tf.reduce_sum(dis1 +dis2) / 2. - def triplet_loss(anchor, positive, negative, margin=1.0): +def triplet_loss(anchor, positive, negative, margin=1.0): """ Triplet Loss. Computes the triplet loss between y_pred (logits) amd @@ -295,10 +295,9 @@ def triplet_loss(anchor, positive, negative, margin=1.0): positive: `Tensor`. Same class as anchor negative: `Tensor`. Different class from anchor margin: . A self-set parameters that indicate the distance between the expected different identity features - """ - - with tf.name_scope("TripletLoss"): - dist1_postive = tf.math.reduce_sum(tf.math.pow((anchor - positive), 2)) - dist2_negative = tf.math.reduce_sum(tf.math.pow((anchor - negative), 2)) - loss = tf.nn.relu(dist1_positive - dist2_negative + margin) - return loss + """ + with tf.name_scope("TripletLoss"): + dist1_postive = tf.math.reduce_sum(tf.math.pow((anchor - positive), 2)) + dist2_negative = tf.math.reduce_sum(tf.math.pow((anchor - negative), 2)) + loss = tf.nn.relu(dist1_postive - dist2_negative + margin) + return loss