diff --git a/site/en/tutorials/load_data/images.ipynb b/site/en/tutorials/load_data/images.ipynb new file mode 100644 index 00000000000..cf0ed80c1af --- /dev/null +++ b/site/en/tutorials/load_data/images.ipynb @@ -0,0 +1,1619 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Loading Images.ipynb", + "version": "0.3.2", + "provenance": [], + "private_outputs": true, + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + } + }, + "cells": [ + { + "metadata": { + "deletable": true, + "editable": true, + "id": "ucMoYase6URl", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "# Load images with `tf.data`" + ] + }, + { + "metadata": { + "id": "_Wwu5SXZmEkB", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "\n", + " \n", + " \n", + " \n", + "
\n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
" + ] + }, + { + "metadata": { + "id": "Oxw4WahM7DU9", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This tutorial provides a simple example of how to load an image dataset using `tf.data`.\n", + "\n", + "The dataset used in this example is distributed as directories of images, with one class of image per directory." + ] + }, + { + "metadata": { + "deletable": true, + "editable": true, + "id": "hoQQiZDB6URn", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Setup" + ] + }, + { + "metadata": { + "id": "QGXxBuPyKJw1", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "!pip install tf-nightly" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "DHz3JONNEHlj", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import tensorflow as tf\n", + "tf.enable_eager_execution()\n", + "tf.VERSION" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "KT6CcaqgQewg", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "AUTOTUNE = tf.data.experimental.AUTOTUNE" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "rxndJHNC8YPM", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Download and inspect the dataset" + ] + }, + { + "metadata": { + "deletable": true, + "editable": true, + "id": "wO0InzL66URu", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Retrieve the images\n", + "\n", + "Before you start any training, you'll need a set of images to teach the network about the new classes you want to recognize. We've created an archive of creative-commons licensed flower photos to use initially. " + ] + }, + { + "metadata": { + "id": "rN-Pc6Zd6awg", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import pathlib\n", + "data_root = tf.keras.utils.get_file('flower_photos','https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz', untar=True)\n", + "data_root = pathlib.Path(data_root)\n", + "print(data_root)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "rFkFK74oO--g", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "After downloading 218MB, you should now have a copy of the flower photos available:" + ] + }, + { + "metadata": { + "id": "7onR_lWE7Njj", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "for item in data_root.iterdir():\n", + " print(item)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "4yYX3ZRqGOuq", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import random\n", + "all_image_paths = list(data_root.glob('*/*'))\n", + "all_image_paths = [str(path) for path in all_image_paths]\n", + "random.shuffle(all_image_paths)\n", + "\n", + "image_count = len(all_image_paths)\n", + "image_count" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vkM-IpB-6URx", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Inspect the images\n", + "Now let's have a quick look at a couple of the images, so we know what we're dealing with:" + ] + }, + { + "metadata": { + "id": "wNGateQJ6UR1", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "attributions = (data_root/\"LICENSE.txt\").read_text().splitlines()[4:]\n", + "attributions = [line.split(' CC-BY') for line in attributions]\n", + "attributions = dict(attributions)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jgowG2xu88Io", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import IPython.display as display\n", + "\n", + "def caption_image(image_path):\n", + " image_rel = pathlib.Path(image_path).relative_to(data_root)\n", + " return \"Image (CC BY 2.0) \" + ' - '.join(attributions[str(image_rel)].split(' - ')[:-1])\n", + " " + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "YIjLi-nX0txI", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "for n in range(3):\n", + " image_path = random.choice(all_image_paths)\n", + " display.display(display.Image(image_path))\n", + " print(caption_image(image_path))\n", + " print()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "OaNOr-co3WKk", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Determine the label for each image" + ] + }, + { + "metadata": { + "id": "-weOQpDw2Jnu", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "List the available labels:" + ] + }, + { + "metadata": { + "id": "ssUZ7Qh96UR3", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())\n", + "label_names" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "9l_JEBql2OzS", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Assign an index to each label:" + ] + }, + { + "metadata": { + "id": "Y8pCV46CzPlp", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "label_to_index = dict((name, index) for index,name in enumerate(label_names))\n", + "label_to_index" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "VkXsHg162T9F", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Create a list of every file, and its label index" + ] + }, + { + "metadata": { + "id": "q62i1RBP4Q02", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "all_image_labels = [label_to_index[pathlib.Path(path).parent.name]\n", + " for path in all_image_paths]\n", + "\n", + "print(\"First 10 labels indices: \", all_image_labels[:10])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "i5L09icm9iph", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Load and format the images" + ] + }, + { + "metadata": { + "id": "SbqqRUS79ooq", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "TensorFlow includes all the tools you need to load and process images:" + ] + }, + { + "metadata": { + "id": "jQZdySHvksOu", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "img_path = all_image_paths[0]\n", + "img_path" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2t2h2XCcmK1Y", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "here is the raw data:" + ] + }, + { + "metadata": { + "id": "LJfkyC_Qkt7A", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "img_raw = tf.read_file(img_path)\n", + "print(repr(img_raw)[:100]+\"...\")" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "opN8AVc8mSbz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Decode it into an image tensor:" + ] + }, + { + "metadata": { + "id": "Tm0tdrlfk0Bb", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "img_tensor = tf.image.decode_image(img_raw)\n", + "\n", + "print(img_tensor.shape)\n", + "print(img_tensor.dtype)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "3k-Of2Tfmbeq", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Resize it for your model:" + ] + }, + { + "metadata": { + "id": "XFpz-3_vlJgp", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "img_final = tf.image.resize_images(img_tensor, [192, 192])\n", + "img_final = img_final/255.0\n", + "print(img_final.shape)\n", + "print(img_final.numpy().min())\n", + "print(img_final.numpy().max())\n" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "aCsAa4Psl4AQ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Wrap up these up in simple functions for later." + ] + }, + { + "metadata": { + "id": "HmUiZJNU73vA", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def preprocess_image(image):\n", + " image = tf.image.decode_jpeg(image, channels=3)\n", + " image = tf.image.resize_images(image, [192, 192])\n", + " image /= 255.0 # normalize to [0,1] range\n", + "\n", + " return image" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "einETrJnO-em", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def load_and_preprocess_image(path):\n", + " image = tf.read_file(path)\n", + " return preprocess_image(image)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "3brWQcdtz78y", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "image_path = all_image_paths[0]\n", + "label = all_image_labels[0]\n", + "\n", + "plt.imshow(load_and_preprocess_image(img_path))\n", + "plt.grid(False)\n", + "plt.xlabel(caption_image(img_path))\n", + "plt.title(label_names[label].title())\n", + "print()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "n2TCr1TQ8pA3", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Build a `tf.data.Dataset`" + ] + }, + { + "metadata": { + "id": "6H9Z5Mq63nSH", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### A dataset of images" + ] + }, + { + "metadata": { + "id": "GN-s04s-6Luq", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The easiest way to build a `tf.data.Dataset` is using the `from_tensor_slices` method.\n", + "\n", + "Slicing the array of strings, results in a dataset of strings:" + ] + }, + { + "metadata": { + "id": "6oRPG3Jz3ie_", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "uML4JeMmIAvO", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The `output_shapes` and `output_types` fields describe the content of each item in the dataset. In this case it is a set of scalar binary-strings" + ] + }, + { + "metadata": { + "id": "mIsNflFbIK34", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "print('shape: ', repr(path_ds.output_shapes))\n", + "print('type: ', path_ds.output_types)\n", + "print()\n", + "print(path_ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "ZjyGcM8OwBJ2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now create a new dataset that loads and formats images on the fly by mapping `preprocess_image` over the dataset of paths." + ] + }, + { + "metadata": { + "id": "D1iba6f4khu-", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=AUTOTUNE)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "JLUPs2a-lEEJ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "plt.figure(figsize=(8,8))\n", + "for n,image in enumerate(image_ds.take(4)):\n", + " plt.subplot(2,2,n+1)\n", + " plt.imshow(image)\n", + " plt.grid(False)\n", + " plt.xticks([])\n", + " plt.yticks([])\n", + " plt.xlabel(caption_image(all_image_paths[n]))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "P6FNqPbxkbdx", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### A dataset of `(image, label)` pairs" + ] + }, + { + "metadata": { + "id": "YgvrWLKG67-x", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Using the same `from_tensor_slices` method we can build a dataset of labels" + ] + }, + { + "metadata": { + "id": "AgBsAiV06udj", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "HEsk5nN0vyeX", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "for label in label_ds.take(10):\n", + " print(label_names[label.numpy()])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jHjgrEeTxyYz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Since the datasets are in the same order we can just zip them together to get a dataset of `(image, label)` pairs." + ] + }, + { + "metadata": { + "id": "AOEWNMdQwsbN", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "yA2F09SJLMuM", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The new dataset's `shapes` and `types` are tuples of shapes and types as well, describing each field:" + ] + }, + { + "metadata": { + "id": "DuVYNinrLL-N", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "print('image shape: ', image_label_ds.output_shapes[0])\n", + "print('label shape: ', image_label_ds.output_shapes[1])\n", + "print('types: ', image_label_ds.output_types)\n", + "print()\n", + "print(image_label_ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "2WYMikoPWOQX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Note: When you have arrays like `all_image_labels` and `all_image_paths` an alternative to `tf.data.dataset.Dataset.zip` is to slice the pair of arrays." + ] + }, + { + "metadata": { + "id": "HOFwZI-2WhzV", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))\n", + "\n", + "# The tuples are unpacked into the positional arguments of the mapped function \n", + "def load_and_preprocess_from_path_label(path, label):\n", + " return load_and_preprocess_image(path), label\n", + "\n", + "image_label_ds = ds.map(load_and_preprocess_from_path_label)\n", + "image_label_ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vYGCgJuR_9Qp", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Basic methods for training" + ] + }, + { + "metadata": { + "id": "wwZavzgsIytz", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "To train a model with this dataset you will want the data:\n", + "\n", + "* To be well shuffled.\n", + "* To be batched.\n", + "* To repeat forever.\n", + "* Batches to be available as soon as possible.\n", + "\n", + "These features can be easily added using the `tf.data` api." + ] + }, + { + "metadata": { + "id": "uZmZJx8ePw_5", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "BATCH_SIZE = 32\n", + "\n", + "# Setting a shuffle buffer size as large as the dataset ensures that the data is\n", + "# completely shuffled.\n", + "ds = image_label_ds.shuffle(buffer_size=image_count)\n", + "ds = ds.repeat()\n", + "ds = ds.batch(BATCH_SIZE)\n", + "# `prefetch` lets the dataset fetch batches, in the background while the model is training.\n", + "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "6JsM-xHiFCuW", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "There are a few things to note here:\n", + "\n", + "1. The order is important. \n", + "\n", + " * A `.shuffle` before a `.repeat` would shuffle items across epoch boundaries (some items will ve seen twice before others are seen at all). \n", + " * A `.shuffle` after a `.batch` would shuffle the order of the batches, but not shuffle the items across batches.\n", + "\n", + "1. We use a `buffer_size` the same size as the dataset for a full shuffle. Up to the dataset size, large values provide better randomization, but use more memory. \n", + "\n", + "1. The shuffle buffer is filled before any elements are pulled from it. So a large `buffer_size` may cause a delay when your `Dataset` is starting.\n", + "\n", + "1. The shuffeled dataset doesn't report the end of a dataset until the shuffle-buffer is completely empty. The `Dataset` is restarted by `.repeat`, causing another wait for the shuffle-buffer to be filled.\n", + "\n", + "This last point can be addressed by using the `tf.data.Dataset.apply` method with the fused `tf.data.experimental.shuffle_and_repeat` function:" + ] + }, + { + "metadata": { + "id": "Ocr6PybXNDoO", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = image_label_ds.apply(\n", + " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", + "ds = ds.batch(BATCH_SIZE)\n", + "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "GBBZMSuAmQVL", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Pipe the dataset to a model\n", + "\n", + "Fetch a copy of MobileNet v2 from `tf.keras.applications`.\n", + "\n", + "This will be used for a simple transfer learning example.\n", + "\n", + "Set the MobileNet weights to be non-trainable:" + ] + }, + { + "metadata": { + "id": "KbJrXn9omO_g", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "mobile_net = tf.keras.applications.MobileNetV2(input_shape=(192, 192, 3), include_top=False)\n", + "mobile_net.trainable=False" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "Y7NVWiLF3Vbf", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This model expects its input to be normalized to the `[-1,1]` range:\n", + "\n", + "```\n", + "help(keras_applications.mobilenet_v2.preprocess_input)\n", + "```\n", + "\n", + "
\n",
+        "...\n",
+        "This function applies the \"Inception\" preprocessing which converts\n",
+        "the RGB values from [0, 255] to [-1, 1] \n",
+        "...\n",
+        "
" + ] + }, + { + "metadata": { + "id": "CboYya6LmdQI", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "So before the passing it to the MobilNet model, we need to convert the input from a range of `[0,1]` to `[-1,1]`." + ] + }, + { + "metadata": { + "id": "SNOkHUGv3FYq", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "def change_range(image,label):\n", + " return 2*image-1, label\n", + "\n", + "keras_ds = ds.map(change_range)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "QDzZ3Nye5Rpv", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The MobileNet returns a `6x6` spatial grid of features for each image.\n", + "\n", + "Pass it a batch of images to see:" + ] + }, + { + "metadata": { + "id": "OzAhGkEK6WuE", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "# The dataset may take a few seconds to start, as it fills its shuffle buffer.\n", + "image_batch, label_batch = next(iter(keras_ds))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "LcFdiWpO5WbV", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "feature_map_batch = mobile_net(image_batch)\n", + "print(feature_map_batch.shape)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "vrbjEvaC5XmU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "So build a model wrapped around MobileNet, and use `tf.keras.layers.GlobalAveragePooling2D` to average over those space dimensions, before the output `tf.keras.layers.Dense` layer:" + ] + }, + { + "metadata": { + "id": "X0ooIU9fNjPJ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model = tf.keras.Sequential([\n", + " mobile_net,\n", + " tf.keras.layers.GlobalAveragePooling2D(),\n", + " tf.keras.layers.Dense(len(label_names))])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "foQYUJs97V4V", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now it produces outputs of the expected shape:" + ] + }, + { + "metadata": { + "id": "1nwYxvpj7ZEf", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "logit_batch = model(image_batch).numpy()\n", + "\n", + "print(\"min logit:\", logit_batch.min())\n", + "print(\"max logit:\", logit_batch.max())\n", + "print()\n", + "\n", + "print(\"Shape:\", logit_batch.shape)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "pFc4I_J2nNOJ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Compile the model to describe the training procedure:" + ] + }, + { + "metadata": { + "id": "ZWGqLEWYRNvv", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model.compile(optimizer=tf.train.AdamOptimizer(), \n", + " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", + " metrics=[\"accuracy\"])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "tF1mO6haBOSd", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "There are 2 trainable variables: the Dense `weights` and `bias`:" + ] + }, + { + "metadata": { + "id": "pPQ5yqyKBJMm", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "len(model.trainable_variables) " + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "kug5Wg66UJjl", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model.summary()" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "f_glpYZ-nYC_", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Train the model.\n", + "\n", + "Normally you would specify the real number of steps per epoch, but for demonstration purposes only run 3 steps." + ] + }, + { + "metadata": { + "id": "AnXPRNWoTypI", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "steps_per_epoch=tf.ceil(len(all_image_paths)/BATCH_SIZE).numpy()\n", + "steps_per_epoch" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "q_8sabaaSGAp", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "model.fit(ds, epochs=1, steps_per_epoch=3)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "UMVnoBcG_NlQ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "## Performance\n", + "\n", + "Note: This section just shows a couple of easy tricks that may help performance. For an in depth guide see [Input Pipeline Performance](https://www.tensorflow.org/guide/performance/datasets).\n", + "\n", + "The simple pipeline used above reads each file individually, on each epoch. This is fine for local training on CPU but may not be sufficient for GPU training, and is totally inappropriate for any sort of distributed training. " + ] + }, + { + "metadata": { + "id": "oNmQqgGhLWie", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "To investigate, first build a simple function to check the performance of our datasets:" + ] + }, + { + "metadata": { + "id": "_gFVe1rp_MYr", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "import time\n", + "\n", + "def timeit(ds, batches=2*steps_per_epoch+1):\n", + " overall_start = time.time()\n", + " # Fetch a single batch to prime the pipeline (fill the shuffle buffer),\n", + " # before starting the timer\n", + " it = iter(ds.take(batches+1))\n", + " next(it)\n", + "\n", + " start = time.time()\n", + " for i,(images,labels) in enumerate(it):\n", + " if i%10 == 0:\n", + " print('.',end='')\n", + " print()\n", + " end = time.time()\n", + "\n", + " duration = end-start\n", + " print(\"{} batches: {} s\".format(batches, duration))\n", + " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*batches/duration))\n", + " print(\"Total time: {}s\".format(end-overall_start))" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "TYiOr4vdLcNX", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The performance of the current dataset is:" + ] + }, + { + "metadata": { + "id": "ZDxLwMJOReVe", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = image_label_ds.apply(\n", + " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", + "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "IjouTJadRxyp", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "HsLlXMO7EWBR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### Cache" + ] + }, + { + "metadata": { + "id": "lV1NOn2zE2lR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Use `tf.data.Dataset.cache` to easily cache calculations across epochs. This is especially performant if the dataq fits in memory.\n", + "\n", + "Here the images are cached, after being pre-precessed (decoded and resized):" + ] + }, + { + "metadata": { + "id": "qj_U09xpDvOg", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = image_label_ds.cache()\n", + "ds = ds.apply(\n", + " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", + "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "rdxpvQ7VEo3y", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "usIv7MqqZQps", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "One disadvantage to using an in memory cache is that the cache must be rebuilt on each run, giving the same startup delay each time the dataset is started:" + ] + }, + { + "metadata": { + "id": "eKX6ergKb_xd", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "jUzpG4lYNkN-", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "If the data doesn't fit in memory, use a cache file:" + ] + }, + { + "metadata": { + "id": "vIvF8K4GMq0g", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = image_label_ds.cache(filename='./cache.tf-data')\n", + "ds = ds.apply(\n", + " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", + "ds = ds.batch(BATCH_SIZE).prefetch(1)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "eTIj6IOmM4yA", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "qqo3dyB0Z4t2", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "The cache file also has the advantage that it can be used to quickly restart the dataset without rebuilding the cache. Note how much faster it is the second time:" + ] + }, + { + "metadata": { + "id": "hZhVdR8MbaUj", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "WqOVlf8tFrDU", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "### TFRecord File" + ] + }, + { + "metadata": { + "id": "y1llOTwWFzmR", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Raw image data\n", + "\n", + "TFRecord files are a simple format to store a sequence of binary blobs. By packing multiple examples into the same file, TensorFlow is able to read multiple examples at once, which is especially important for performance when using a remote storage service such as GCS.\n", + "\n", + "First, build a TFRecord file from the raw image data:" + ] + }, + { + "metadata": { + "id": "EqtARqKuHQLu", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "image_ds = tf.data.Dataset.from_tensor_slices(all_image_paths).map(tf.read_file)\n", + "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", + "tfrec.write(image_ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "flR2GXWFKcO1", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Next build a dataset that reads from the TFRecord file and decodes/reformats the images using the `preprocess_image` function we defined earlier." + ] + }, + { + "metadata": { + "id": "j9PVUL2SFufn", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "image_ds = tf.data.TFRecordDataset('images.tfrec').map(preprocess_image)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "cRp1eZDRKzyN", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Zip that with the labels dataset we defined earlier, to get the expected `(image,label)` pairs." + ] + }, + { + "metadata": { + "id": "7XI_nDU2KuhS", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = tf.data.Dataset.zip((image_ds, label_ds))\n", + "ds = ds.apply(\n", + " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", + "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "3ReSapoPK22E", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "wb7VyoKNOMms", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "This is slower than the `cache` version because we have not cached the preprocessing." + ] + }, + { + "metadata": { + "id": "NF9W-CTKkM-f", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "#### Serialized Tensors" + ] + }, + { + "metadata": { + "id": "J9HzljSPkxt0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "To save some preprocessing to the TFRecord file, first make a dataset of the processed images, as before:" + ] + }, + { + "metadata": { + "id": "OzS0Azukkjyw", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "paths_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)\n", + "image_ds = paths_ds.map(load_and_preprocess_image)\n", + "image_ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "onWOwLpYlzJQ", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now instead of a dataset of `.jpeg` strings, this is a dataset of tensors.\n", + "\n", + "To serialize this to a TFRecord file you first convert the dataset of tensors to a dataset of strings." + ] + }, + { + "metadata": { + "id": "xxZSwnRllyf0", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = image_ds.map(tf.serialize_tensor)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "w9N6hJWAkKPC", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", + "tfrec.write(ds)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "OlFc9dJSmcx0", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "With the preprocessing cached, data can be loaded from the TFrecord file quite efficiently. Just remember to de-serialized tensor before trying to use it." + ] + }, + { + "metadata": { + "id": "BsqFyTBFmSCZ", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "RESTORE_TYPE = image_ds.output_types\n", + "RESTORE_SHAPE = image_ds.output_shapes\n", + "\n", + "ds = tf.data.TFRecordDataset('images.tfrec')\n", + "\n", + "def parse(x):\n", + " result = tf.parse_tensor(x, out_type=RESTORE_TYPE)\n", + " result = tf.reshape(result, RESTORE_SHAPE)\n", + " return result\n", + "\n", + "ds = ds.map(parse, num_parallel_calls=AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "OPs_sLV9pQg5", + "colab_type": "text" + }, + "cell_type": "markdown", + "source": [ + "Now, add the labels and apply the same standard operations as before: " + ] + }, + { + "metadata": { + "id": "XYxBwaLYnGop", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "ds = tf.data.Dataset.zip((ds, label_ds))\n", + "ds = ds.apply(\n", + " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", + "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", + "ds" + ], + "execution_count": 0, + "outputs": [] + }, + { + "metadata": { + "id": "W8X6RmGan1-P", + "colab_type": "code", + "colab": {} + }, + "cell_type": "code", + "source": [ + "timeit(ds)" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file