MNIST with AdaNet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# USING AdaNet on MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install adanet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import functools\n",
    "\n",
    "import adanet\n",
    "from adanet.examples import simple_dnn\n",
    "import tensorflow as tf\n",
    "\n",
    "# The random seed to use.\n",
    "RANDOM_SEED = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 16s 1us/step\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = (\n",
    "    tf.keras.datasets.mnist.load_data())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "FEATURES_KEY = \"images\"\n",
    "\n",
    "\n",
    "def generator(images, labels):\n",
    "  \"\"\"Returns a generator that returns image-label pairs.\"\"\"\n",
    "\n",
    "  def _gen():\n",
    "    for image, label in zip(images, labels):\n",
    "      yield image, label\n",
    "\n",
    "  return _gen\n",
    "\n",
    "\n",
    "def preprocess_image(image, label):\n",
    "  \"\"\"Preprocesses an image for an `Estimator`.\"\"\"\n",
    "  # Pixel degerleri 0 ve 1 arasinda olmali.\n",
    "  image = image / 255.\n",
    "  # 2D-convolution uygulamak icin resimleri yeniden bicimlendirmeliyiz.\n",
    "  image = tf.reshape(image, [28, 28, 1])\n",
    "  # Ozelliklerimizi bir dictonary objesinde toparlamaliyiz.\n",
    "  features = {FEATURES_KEY: image}\n",
    "  return features, label\n",
    "\n",
    "\n",
    "def input_fn(partition, training, batch_size):\n",
    "  \"\"\"Generate an input_fn for the Estimator.\"\"\"\n",
    "\n",
    "  def _input_fn():\n",
    "    if partition == \"train\":\n",
    "      dataset = tf.data.Dataset.from_generator(\n",
    "          generator(x_train, y_train), (tf.float32, tf.int32), ((28, 28), ()))\n",
    "    else:\n",
    "      dataset = tf.data.Dataset.from_generator(\n",
    "          generator(x_test, y_test), (tf.float32, tf.int32), ((28, 28), ()))\n",
    "\n",
    "    \n",
    "    if training:\n",
    "      dataset = dataset.shuffle(10 * batch_size, seed=RANDOM_SEED).repeat()\n",
    "\n",
    "    dataset = dataset.map(preprocess_image).batch(batch_size)\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    features, labels = iterator.get_next()\n",
    "    return features, labels\n",
    "\n",
    "  return _input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Siniflandirma ogelerinin sayisi.\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# Gradyanlar hesaplanirken her mini-batch uzerinde ortalama loss hesaplanacak.\n",
    "loss_reduction = tf.losses.Reduction.SUM_OVER_BATCH_SIZE\n",
    "\n",
    "# `Head`, `Estimator`lar icin loss function ve metrikleri tanimliyor.\n",
    "head = tf.contrib.estimator.multi_class_head(\n",
    "    NUM_CLASSES, loss_reduction=loss_reduction)\n",
    "\n",
    "# Bazi `Estimator`lar girdi ozelliklerini anlamak icin ozellik sutunlari kullaniyor.\n",
    "feature_columns = [\n",
    "    tf.feature_column.numeric_column(FEATURES_KEY, shape=[28, 28, 1])\n",
    "]\n",
    "\n",
    "# Estimator konfigurasyonu.\n",
    "config = tf.estimator.RunConfig(\n",
    "    save_checkpoints_steps=50000,\n",
    "    save_summary_steps=50000,\n",
    "    tf_random_seed=RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Using temporary folder as model directory: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp2ktdva5_\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'C:\\\\Users\\\\dell\\\\AppData\\\\Local\\\\Temp\\\\tmp2ktdva5_', '_tf_random_seed': 42, '_save_summary_steps': 50000, '_save_checkpoints_steps': 50000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x00000243B40E8E48>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 50000 or save_checkpoints_secs None.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp2ktdva5_\\model.ckpt.\n",
      "INFO:tensorflow:loss = 2.3025854, step = 1\n",
      "INFO:tensorflow:global_step/sec: 62.2074\n",
      "INFO:tensorflow:loss = 1.3225186, step = 101 (1.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 80.1255\n",
      "INFO:tensorflow:loss = 0.6913334, step = 201 (1.248 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.8571\n",
      "INFO:tensorflow:loss = 0.37077364, step = 301 (1.169 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.5047\n",
      "INFO:tensorflow:loss = 0.56232077, step = 401 (1.166 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5092\n",
      "INFO:tensorflow:loss = 0.31495035, step = 501 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5306\n",
      "INFO:tensorflow:loss = 0.21376613, step = 601 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 80.669\n",
      "INFO:tensorflow:loss = 0.27821153, step = 701 (1.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.9359\n",
      "INFO:tensorflow:loss = 0.33467183, step = 801 (1.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5306\n",
      "INFO:tensorflow:loss = 0.29745227, step = 901 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.7459\n",
      "INFO:tensorflow:loss = 0.30318943, step = 1001 (1.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5307\n",
      "INFO:tensorflow:loss = 0.32642648, step = 1101 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.2322\n",
      "INFO:tensorflow:loss = 0.34919763, step = 1201 (1.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.8876\n",
      "INFO:tensorflow:loss = 0.21634534, step = 1301 (1.272 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.6417\n",
      "INFO:tensorflow:loss = 0.3045628, step = 1401 (1.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.457\n",
      "INFO:tensorflow:loss = 0.23112139, step = 1501 (1.232 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.9027\n",
      "INFO:tensorflow:loss = 0.42135483, step = 1601 (1.217 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.4946\n",
      "INFO:tensorflow:loss = 0.34508914, step = 1701 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.73\n",
      "INFO:tensorflow:loss = 0.24383737, step = 1801 (1.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.6566\n",
      "INFO:tensorflow:loss = 0.28901017, step = 1901 (1.150 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.6395\n",
      "INFO:tensorflow:loss = 0.23797704, step = 2001 (1.272 sec)\n",
      "INFO:tensorflow:global_step/sec: 79.8602\n",
      "INFO:tensorflow:loss = 0.24146298, step = 2101 (1.256 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.263\n",
      "INFO:tensorflow:loss = 0.33190995, step = 2201 (1.227 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.8059\n",
      "INFO:tensorflow:loss = 0.2553122, step = 2301 (1.208 sec)\n",
      "INFO:tensorflow:global_step/sec: 88.9487\n",
      "INFO:tensorflow:loss = 0.20851849, step = 2401 (1.124 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.4095\n",
      "INFO:tensorflow:loss = 0.21551138, step = 2501 (1.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 80.93\n",
      "INFO:tensorflow:loss = 0.23730999, step = 2601 (1.232 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.0047\n",
      "INFO:tensorflow:loss = 0.3084969, step = 2701 (1.194 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.1692\n",
      "INFO:tensorflow:loss = 0.17739734, step = 2801 (1.157 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.9358\n",
      "INFO:tensorflow:loss = 0.40055996, step = 2901 (1.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.8313\n",
      "INFO:tensorflow:loss = 0.3638569, step = 3001 (1.152 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5306\n",
      "INFO:tensorflow:loss = 0.19954783, step = 3101 (1.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.5305\n",
      "INFO:tensorflow:loss = 0.43056262, step = 3201 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.3709\n",
      "INFO:tensorflow:loss = 0.5477644, step = 3301 (1.158 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.8023\n",
      "INFO:tensorflow:loss = 0.3661342, step = 3401 (1.161 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.8165\n",
      "INFO:tensorflow:loss = 0.18359986, step = 3501 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 82.384\n",
      "INFO:tensorflow:loss = 0.20303293, step = 3601 (1.210 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.6813\n",
      "INFO:tensorflow:loss = 0.24621975, step = 3701 (1.154 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.2324\n",
      "INFO:tensorflow:loss = 0.46217203, step = 3801 (1.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.8311\n",
      "INFO:tensorflow:loss = 0.2373266, step = 3901 (1.156 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.936\n",
      "INFO:tensorflow:loss = 0.27906618, step = 4001 (1.160 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.936\n",
      "INFO:tensorflow:loss = 0.2575124, step = 4101 (1.164 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.9172\n",
      "INFO:tensorflow:loss = 0.5643625, step = 4201 (1.192 sec)\n",
      "INFO:tensorflow:global_step/sec: 81.4572\n",
      "INFO:tensorflow:loss = 0.46703285, step = 4301 (1.232 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.9657\n",
      "INFO:tensorflow:loss = 0.3961224, step = 4401 (1.177 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.4844\n",
      "INFO:tensorflow:loss = 0.4562727, step = 4501 (1.180 sec)\n",
      "INFO:tensorflow:global_step/sec: 86.7063\n",
      "INFO:tensorflow:loss = 0.34617066, step = 4601 (1.153 sec)\n",
      "INFO:tensorflow:global_step/sec: 84.9351\n",
      "INFO:tensorflow:loss = 0.42493618, step = 4701 (1.181 sec)\n",
      "INFO:tensorflow:global_step/sec: 83.9173\n",
      "INFO:tensorflow:loss = 0.2685781, step = 4801 (1.188 sec)\n",
      "INFO:tensorflow:global_step/sec: 85.936\n",
      "INFO:tensorflow:loss = 0.22215945, step = 4901 (1.168 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5000 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp2ktdva5_\\model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-10-19:38:59\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp2ktdva5_\\model.ckpt-5000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-10-19:39:02\n",
      "INFO:tensorflow:Saving dict for global step 5000: accuracy = 0.9258, average_loss = 0.27285188, global_step = 5000, loss = 0.27215475\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5000: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp2ktdva5_\\model.ckpt-5000\n",
      "INFO:tensorflow:Loss for final step: 0.13550904.\n",
      "Accuracy: 0.9258\n",
      "Loss: 0.27285188\n"
     ]
    }
   ],
   "source": [
    "# Once basit bir lineer model egitelim\n",
    "LEARNING_RATE = 0.001  \n",
    "TRAIN_STEPS = 5000  \n",
    "BATCH_SIZE = 64  \n",
    "\n",
    "estimator = tf.estimator.LinearClassifier(\n",
    "    feature_columns=feature_columns,\n",
    "    n_classes=NUM_CLASSES,\n",
    "    optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),\n",
    "    loss_reduction=loss_reduction,\n",
    "    config=config)\n",
    "\n",
    "results, _ = tf.estimator.train_and_evaluate(\n",
    "    estimator,\n",
    "    train_spec=tf.estimator.TrainSpec(\n",
    "        input_fn=input_fn(\"train\", training=True, batch_size=BATCH_SIZE),\n",
    "        max_steps=TRAIN_STEPS),\n",
    "    eval_spec=tf.estimator.EvalSpec(\n",
    "        input_fn=input_fn(\"test\", training=False, batch_size=BATCH_SIZE),\n",
    "        steps=None))\n",
    "print(\"Accuracy:\", results[\"accuracy\"])\n",
    "print(\"Loss:\", results[\"average_loss\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "global step 5000: accuracy = 0.9258, average_loss = 0.27285188, global_step = 5000, loss = 0.27215475\n",
    "Loss for final step: 0.13550904.\n",
    "Accuracy: 0.9258\n",
    "Loss: 0.27285188"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Varsayilan degerler kullanarak olusturulan lineer model yaklasik %92.58 dogruluk degeri elde etti.\n",
    "\n",
    "Bakalim AdaNet simple_dnn modelini kullanarak daha iyi bir sonuc elde edebilecek miyiz?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Using temporary folder as model directory: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'C:\\\\Users\\\\dell\\\\AppData\\\\Local\\\\Temp\\\\tmp675d5x3w', '_tf_random_seed': 42, '_save_summary_steps': 50000, '_save_checkpoints_steps': 50000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x00000243B4452E48>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 50000 or save_checkpoints_secs None.\n",
      "INFO:tensorflow:Beginning training AdaNet iteration 0\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt.\n",
      "INFO:tensorflow:loss = 2.3048944, step = 1\n",
      "INFO:tensorflow:global_step/sec: 71.6259\n",
      "INFO:tensorflow:loss = 0.6711568, step = 101 (1.397 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.6614\n",
      "INFO:tensorflow:loss = 0.31763986, step = 201 (1.045 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.5698\n",
      "INFO:tensorflow:loss = 0.1615103, step = 301 (1.046 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.7553\n",
      "INFO:tensorflow:loss = 0.32465225, step = 401 (1.055 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.9364\n",
      "INFO:tensorflow:loss = 0.14924046, step = 501 (1.042 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.845\n",
      "INFO:tensorflow:loss = 0.07197814, step = 601 (1.054 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.7413\n",
      "INFO:tensorflow:loss = 0.088311166, step = 701 (1.114 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.5698\n",
      "INFO:tensorflow:loss = 0.10245603, step = 801 (1.046 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.2061\n",
      "INFO:tensorflow:loss = 0.0825947, step = 901 (1.050 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.2129\n",
      "INFO:tensorflow:loss = 0.29055962, step = 1001 (1.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.7528\n",
      "INFO:tensorflow:loss = 0.11370947, step = 1101 (1.044 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.0284\n",
      "INFO:tensorflow:loss = 0.088960156, step = 1201 (1.041 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.8444\n",
      "INFO:tensorflow:loss = 0.052612334, step = 1301 (1.043 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.2061\n",
      "INFO:tensorflow:loss = 0.16187888, step = 1401 (1.050 sec)\n",
      "INFO:tensorflow:global_step/sec: 96.3057\n",
      "INFO:tensorflow:loss = 0.081651606, step = 1501 (1.038 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.2967\n",
      "INFO:tensorflow:loss = 0.24000263, step = 1601 (1.049 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.6612\n",
      "INFO:tensorflow:loss = 0.11327048, step = 1701 (1.045 sec)\n",
      "INFO:tensorflow:global_step/sec: 78.2954\n",
      "INFO:tensorflow:loss = 0.12082756, step = 1801 (1.276 sec)\n",
      "INFO:tensorflow:global_step/sec: 79.477\n",
      "INFO:tensorflow:loss = 0.105430275, step = 1901 (1.259 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.9351\n",
      "INFO:tensorflow:loss = 0.029110946, step = 2001 (1.053 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.7528\n",
      "INFO:tensorflow:loss = 0.11788218, step = 2101 (1.044 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.4785\n",
      "INFO:tensorflow:loss = 0.14209202, step = 2201 (1.047 sec)\n",
      "INFO:tensorflow:global_step/sec: 94.0428\n",
      "INFO:tensorflow:loss = 0.1702203, step = 2301 (1.063 sec)\n",
      "INFO:tensorflow:global_step/sec: 95.5698\n",
      "INFO:tensorflow:loss = 0.08514093, step = 2401 (1.046 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2500 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-10-19:44:43\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-2500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving candidate 'linear' dict for global step 2500: accuracy/adanet/adanet_weighted_ensemble = 0.9238, accuracy/adanet/subnetwork = 0.9238, accuracy/adanet/uniform_average_ensemble = 0.9238, architecture/adanet/ensembles = b\"\\nW\\n6adanet/iteration_0/ensemble_linear/architecture/adanetB\\x13\\x08\\x07\\x12\\x00B\\r| b'linear' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.2854871, average_loss/adanet/subnetwork = 0.2854871, average_loss/adanet/uniform_average_ensemble = 0.2854871, loss/adanet/adanet_weighted_ensemble = 0.2844932, loss/adanet/subnetwork = 0.2844932, loss/adanet/uniform_average_ensemble = 0.2844932\n",
      "INFO:tensorflow:Saving candidate '1_layer_dnn' dict for global step 2500: accuracy/adanet/adanet_weighted_ensemble = 0.957, accuracy/adanet/subnetwork = 0.957, accuracy/adanet/uniform_average_ensemble = 0.957, architecture/adanet/ensembles = b\"\\na\\n;adanet/iteration_0/ensemble_1_layer_dnn/architecture/adanetB\\x18\\x08\\x07\\x12\\x00B\\x12| b'1_layer_dnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.14346279, average_loss/adanet/subnetwork = 0.14346279, average_loss/adanet/uniform_average_ensemble = 0.14346279, loss/adanet/adanet_weighted_ensemble = 0.14280926, loss/adanet/subnetwork = 0.14280926, loss/adanet/uniform_average_ensemble = 0.14280926\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-10-19:44:47\n",
      "INFO:tensorflow:Saving dict for global step 2500: accuracy = 0.957, accuracy/adanet/adanet_weighted_ensemble = 0.957, accuracy/adanet/subnetwork = 0.957, accuracy/adanet/uniform_average_ensemble = 0.957, average_loss = 0.14346279, average_loss/adanet/adanet_weighted_ensemble = 0.14346279, average_loss/adanet/subnetwork = 0.14346279, average_loss/adanet/uniform_average_ensemble = 0.14346279, global_step = 2500, loss = 0.14280926, loss/adanet/adanet_weighted_ensemble = 0.14280926, loss/adanet/subnetwork = 0.14280926, loss/adanet/uniform_average_ensemble = 0.14280926\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2500: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-2500\n",
      "INFO:tensorflow:Loss for final step: 0.09594993.\n",
      "INFO:tensorflow:Starting ensemble evaluation for iteration 0\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-2500\n",
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\adanet\\core\\estimator.py:707: start_queue_runners (from tensorflow.python.training.queue_runner_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "To construct input pipelines, use the `tf.data` module.\n",
      "WARNING:tensorflow:`tf.train.start_queue_runners()` was called when no queue runners were defined. You can safely remove the call to this deprecated function.\n",
      "INFO:tensorflow:Encountered end of input after 939 evaluations\n",
      "INFO:tensorflow:Computed ensemble metrics: adanet_loss/linear = 0.273420, adanet_loss/1_layer_dnn = 0.117683\n",
      "INFO:tensorflow:Finished ensemble evaluation for iteration 0\n",
      "INFO:tensorflow:'1_layer_dnn' at index 1 is moving onto the next iteration\n",
      "INFO:tensorflow:Freezing best ensemble to C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\frozen/ensemble-0.meta\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-2500\n",
      "INFO:tensorflow:Importing frozen ensemble from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\frozen/ensemble-0.meta with features: ['images'].\n",
      "INFO:tensorflow:Overwriting checkpoint with new graph for iteration 1 to C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-2500\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-2500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`tf.train.start_queue_runners()` was called when no queue runners were defined. You can safely remove the call to this deprecated function.\n",
      "INFO:tensorflow:Finished training Adanet iteration 0\n",
      "INFO:tensorflow:Beginning training AdaNet iteration 1\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Importing frozen ensemble from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\frozen/ensemble-0.meta with features: ['images'].\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\increment.ckpt-1\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2500 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt.\n",
      "INFO:tensorflow:loss = 0.10201884, step = 2501\n",
      "INFO:tensorflow:global_step/sec: 63.9774\n",
      "INFO:tensorflow:loss = 0.047877535, step = 2601 (1.563 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.6315\n",
      "INFO:tensorflow:loss = 0.068694524, step = 2701 (1.091 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.0478\n",
      "INFO:tensorflow:loss = 0.110824585, step = 2801 (1.098 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.2227\n",
      "INFO:tensorflow:loss = 0.20001385, step = 2901 (1.084 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.7996\n",
      "INFO:tensorflow:loss = 0.06507363, step = 3001 (1.089 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.6314\n",
      "INFO:tensorflow:loss = 0.010006094, step = 3101 (1.091 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.3805\n",
      "INFO:tensorflow:loss = 0.03259456, step = 3201 (1.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.8839\n",
      "INFO:tensorflow:loss = 0.03803782, step = 3301 (1.088 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.1814\n",
      "INFO:tensorflow:loss = 0.05038324, step = 3401 (1.121 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.393\n",
      "INFO:tensorflow:loss = 0.17537841, step = 3501 (1.082 sec)\n",
      "INFO:tensorflow:global_step/sec: 90.3082\n",
      "INFO:tensorflow:loss = 0.07411386, step = 3601 (1.107 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.2138\n",
      "INFO:tensorflow:loss = 0.059416, step = 3701 (1.096 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.5477\n",
      "INFO:tensorflow:loss = 0.052233107, step = 3801 (1.092 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.0529\n",
      "INFO:tensorflow:loss = 0.09173106, step = 3901 (1.086 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.7155\n",
      "INFO:tensorflow:loss = 0.088061415, step = 4001 (1.090 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.297\n",
      "INFO:tensorflow:loss = 0.2145998, step = 4101 (1.095 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.4639\n",
      "INFO:tensorflow:loss = 0.070446625, step = 4201 (1.093 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.9684\n",
      "INFO:tensorflow:loss = 0.08410535, step = 4301 (1.087 sec)\n",
      "INFO:tensorflow:global_step/sec: 92.053\n",
      "INFO:tensorflow:loss = 0.06114815, step = 4401 (1.086 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.5476\n",
      "INFO:tensorflow:loss = 0.013738263, step = 4501 (1.113 sec)\n",
      "INFO:tensorflow:global_step/sec: 89.6585\n",
      "INFO:tensorflow:loss = 0.10324931, step = 4601 (1.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.7154\n",
      "INFO:tensorflow:loss = 0.0737783, step = 4701 (1.090 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.9684\n",
      "INFO:tensorflow:loss = 0.14220607, step = 4801 (1.087 sec)\n",
      "INFO:tensorflow:global_step/sec: 91.2139\n",
      "INFO:tensorflow:loss = 0.08775904, step = 4901 (1.096 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5000 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Importing frozen ensemble from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\frozen/ensemble-0.meta with features: ['images'].\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-10-19:45:36\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-5000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving candidate 'previous_ensemble' dict for global step 5000: accuracy/adanet/adanet_weighted_ensemble = 0.957, accuracy/adanet/subnetwork = 0.957, accuracy/adanet/uniform_average_ensemble = 0.957, architecture/adanet/ensembles = b\"\\nR\\n,adanet/previous_ensemble/architecture/adanetB\\x18\\x08\\x07\\x12\\x00B\\x12| b'1_layer_dnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.14346279, average_loss/adanet/subnetwork = 0.14346279, average_loss/adanet/uniform_average_ensemble = 0.14346279, loss/adanet/adanet_weighted_ensemble = 0.14280926, loss/adanet/subnetwork = 0.14280926, loss/adanet/uniform_average_ensemble = 0.14280926\n",
      "INFO:tensorflow:Saving candidate '1_layer_dnn' dict for global step 5000: accuracy/adanet/adanet_weighted_ensemble = 0.9639, accuracy/adanet/subnetwork = 0.9609, accuracy/adanet/uniform_average_ensemble = 0.9639, architecture/adanet/ensembles = b\"\\nr\\n;adanet/iteration_1/ensemble_1_layer_dnn/architecture/adanetB)\\x08\\x07\\x12\\x00B#| b'1_layer_dnn' | b'1_layer_dnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.1214199, average_loss/adanet/subnetwork = 0.14007227, average_loss/adanet/uniform_average_ensemble = 0.1214199, loss/adanet/adanet_weighted_ensemble = 0.12084922, loss/adanet/subnetwork = 0.13941163, loss/adanet/uniform_average_ensemble = 0.12084922\n",
      "INFO:tensorflow:Saving candidate '2_layer_dnn' dict for global step 5000: accuracy/adanet/adanet_weighted_ensemble = 0.9642, accuracy/adanet/subnetwork = 0.9609, accuracy/adanet/uniform_average_ensemble = 0.9642, architecture/adanet/ensembles = b\"\\nr\\n;adanet/iteration_1/ensemble_2_layer_dnn/architecture/adanetB)\\x08\\x07\\x12\\x00B#| b'1_layer_dnn' | b'2_layer_dnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.11878315, average_loss/adanet/subnetwork = 0.133459, average_loss/adanet/uniform_average_ensemble = 0.11878315, loss/adanet/adanet_weighted_ensemble = 0.11823941, loss/adanet/subnetwork = 0.13285713, loss/adanet/uniform_average_ensemble = 0.11823941\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-10-19:45:41\n",
      "INFO:tensorflow:Saving dict for global step 5000: accuracy = 0.957, accuracy/adanet/adanet_weighted_ensemble = 0.957, accuracy/adanet/subnetwork = 0.957, accuracy/adanet/uniform_average_ensemble = 0.957, average_loss = 0.14346279, average_loss/adanet/adanet_weighted_ensemble = 0.14346279, average_loss/adanet/subnetwork = 0.14346279, average_loss/adanet/uniform_average_ensemble = 0.14346279, global_step = 5000, loss = 0.14280926, loss/adanet/adanet_weighted_ensemble = 0.14280926, loss/adanet/subnetwork = 0.14280926, loss/adanet/uniform_average_ensemble = 0.14280926\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5000: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-5000\n",
      "INFO:tensorflow:Loss for final step: 0.029671986.\n",
      "Accuracy: 0.957\n",
      "Loss: 0.14346279\n"
     ]
    }
   ],
   "source": [
    "LEARNING_RATE = 0.003  \n",
    "TRAIN_STEPS = 5000  \n",
    "BATCH_SIZE = 64  \n",
    "ADANET_ITERATIONS = 2  \n",
    "\n",
    "estimator = adanet.Estimator(\n",
    "    head=head,\n",
    "    subnetwork_generator=simple_dnn.Generator(\n",
    "        feature_columns=feature_columns,\n",
    "        optimizer=tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE),\n",
    "        seed=RANDOM_SEED),\n",
    "    max_iteration_steps=TRAIN_STEPS // ADANET_ITERATIONS,\n",
    "    evaluator=adanet.Evaluator(\n",
    "        input_fn=input_fn(\"train\", training=False, batch_size=BATCH_SIZE),\n",
    "        steps=None),\n",
    "    config=config)\n",
    "\n",
    "results, _ = tf.estimator.train_and_evaluate(\n",
    "    estimator,\n",
    "    train_spec=tf.estimator.TrainSpec(\n",
    "        input_fn=input_fn(\"train\", training=True, batch_size=BATCH_SIZE),\n",
    "        max_steps=TRAIN_STEPS),\n",
    "    eval_spec=tf.estimator.EvalSpec(\n",
    "        input_fn=input_fn(\"test\", training=False, batch_size=BATCH_SIZE),\n",
    "        steps=None))\n",
    "print(\"Accuracy:\", results[\"accuracy\"])\n",
    "print(\"Loss:\", results[\"average_loss\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving dict for global step 5000: accuracy = 0.957, accuracy/adanet/adanet_weighted_ensemble = 0.957, accuracy/adanet/subnetwork = 0.957, accuracy/adanet/uniform_average_ensemble = 0.957, average_loss = 0.14346279, average_loss/adanet/adanet_weighted_ensemble = 0.14346279, average_loss/adanet/subnetwork = 0.14346279, average_loss/adanet/uniform_average_ensemble = 0.14346279, global_step = 5000, loss = 0.14280926, loss/adanet/adanet_weighted_ensemble = 0.14280926, loss/adanet/subnetwork = 0.14280926, loss/adanet/uniform_average_ensemble = 0.14280926\n",
    "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5000: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp675d5x3w\\model.ckpt-5000\n",
    "INFO:tensorflow:Loss for final step: 0.029671986.\n",
    "Accuracy: 0.957\n",
    "Loss: 0.14346279"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AdaNet'in simple_dnn modeli varsayilan degerlerle basarimi %95.7'ye cikardi."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCNNBuilder(adanet.subnetwork.Builder):\n",
    "  \"\"\"Builds a CNN subnetwork for AdaNet.\"\"\"\n",
    "\n",
    "  def __init__(self, learning_rate, max_iteration_steps, seed):\n",
    "    \"\"\"Initializes a `SimpleCNNBuilder`.\n",
    "\n",
    "    Args:\n",
    "      learning_rate: The float learning rate to use.\n",
    "      max_iteration_steps: The number of steps per iteration.\n",
    "      seed: The random seed.\n",
    "\n",
    "    Returns:\n",
    "      An instance of `SimpleCNNBuilder`.\n",
    "    \"\"\"\n",
    "    self._learning_rate = learning_rate\n",
    "    self._max_iteration_steps = max_iteration_steps\n",
    "    self._seed = seed\n",
    "\n",
    "  def build_subnetwork(self,\n",
    "                       features,\n",
    "                       logits_dimension,\n",
    "                       training,\n",
    "                       iteration_step,\n",
    "                       summary,\n",
    "                       previous_ensemble=None):\n",
    "    \"\"\"See `adanet.subnetwork.Builder`.\"\"\"\n",
    "    images = tf.to_float(features[FEATURES_KEY])\n",
    "    kernel_initializer = tf.keras.initializers.he_normal(seed=self._seed)\n",
    "    x = tf.layers.conv2d(\n",
    "        images,\n",
    "        filters=16,\n",
    "        kernel_size=3,\n",
    "        padding=\"same\",\n",
    "        activation=\"relu\",\n",
    "        kernel_initializer=kernel_initializer)\n",
    "    x = tf.layers.max_pooling2d(x, pool_size=2, strides=2)\n",
    "    x = tf.layers.flatten(x)\n",
    "    x = tf.layers.dense(\n",
    "        x, units=64, activation=\"relu\", kernel_initializer=kernel_initializer)\n",
    "\n",
    "    # The `Head` passed to adanet.Estimator will apply the softmax activation.\n",
    "    logits = tf.layers.dense(\n",
    "        x, units=10, activation=None, kernel_initializer=kernel_initializer)\n",
    "\n",
    "    # Use a constant complexity measure, since all subnetworks have the same\n",
    "    # architecture and hyperparameters.\n",
    "    complexity = tf.constant(1)\n",
    "\n",
    "    return adanet.Subnetwork(\n",
    "        last_layer=x,\n",
    "        logits=logits,\n",
    "        complexity=complexity,\n",
    "        persisted_tensors={})\n",
    "\n",
    "  def build_subnetwork_train_op(self, \n",
    "                                subnetwork, \n",
    "                                loss, \n",
    "                                var_list, \n",
    "                                labels, \n",
    "                                iteration_step,\n",
    "                                summary, \n",
    "                                previous_ensemble=None):\n",
    "    \"\"\"See `adanet.subnetwork.Builder`.\"\"\"\n",
    "\n",
    "    # Momentum optimizer with cosine learning rate decay works well with CNNs.\n",
    "    learning_rate = tf.train.cosine_decay(\n",
    "        learning_rate=self._learning_rate,\n",
    "        global_step=iteration_step,\n",
    "        decay_steps=self._max_iteration_steps)\n",
    "    optimizer = tf.train.MomentumOptimizer(learning_rate, .9)\n",
    "    # NOTE: The `adanet.Estimator` increments the global step.\n",
    "    return optimizer.minimize(loss=loss, var_list=var_list)\n",
    "\n",
    "  def build_mixture_weights_train_op(self, loss, var_list, logits, labels,\n",
    "                                     iteration_step, summary):\n",
    "    \"\"\"See `adanet.subnetwork.Builder`.\"\"\"\n",
    "    return tf.no_op(\"mixture_weights_train_op\")\n",
    "\n",
    "  @property\n",
    "  def name(self):\n",
    "    \"\"\"See `adanet.subnetwork.Builder`.\"\"\"\n",
    "    return \"simple_cnn\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCNNGenerator(adanet.subnetwork.Generator):\n",
    "  \"\"\"Generates a `SimpleCNN` at each iteration.\n",
    "  \"\"\"\n",
    "\n",
    "  def __init__(self, learning_rate, max_iteration_steps, seed=None):\n",
    "    \"\"\"Initializes a `Generator` that builds `SimpleCNNs`.\n",
    "\n",
    "    Args:\n",
    "      learning_rate: The float learning rate to use.\n",
    "      max_iteration_steps: The number of steps per iteration.\n",
    "      seed: The random seed.\n",
    "\n",
    "    Returns:\n",
    "      An instance of `Generator`.\n",
    "    \"\"\"\n",
    "    self._seed = seed\n",
    "    self._dnn_builder_fn = functools.partial(\n",
    "        SimpleCNNBuilder,\n",
    "        learning_rate=learning_rate,\n",
    "        max_iteration_steps=max_iteration_steps)\n",
    "\n",
    "  def generate_candidates(self, previous_ensemble, iteration_number,\n",
    "                          previous_ensemble_reports, all_reports):\n",
    "    \"\"\"See `adanet.subnetwork.Generator`.\"\"\"\n",
    "    seed = self._seed\n",
    "    # Change the seed according to the iteration so that each subnetwork\n",
    "    # learns something different.\n",
    "    if seed is not None:\n",
    "      seed += iteration_number\n",
    "    return [self._dnn_builder_fn(seed=seed)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Using temporary folder as model directory: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'C:\\\\Users\\\\dell\\\\AppData\\\\Local\\\\Temp\\\\tmp3rlexm4i', '_tf_random_seed': 42, '_save_summary_steps': 50000, '_save_checkpoints_steps': 50000, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x00000243BCA3C4A8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 50000 or save_checkpoints_secs None.\n",
      "INFO:tensorflow:Beginning training AdaNet iteration 0\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt.\n",
      "INFO:tensorflow:loss = 2.3571274, step = 1\n",
      "INFO:tensorflow:global_step/sec: 30.734\n",
      "INFO:tensorflow:loss = 0.24705148, step = 101 (3.254 sec)\n",
      "INFO:tensorflow:global_step/sec: 31.1815\n",
      "INFO:tensorflow:loss = 0.12670697, step = 201 (3.207 sec)\n",
      "INFO:tensorflow:global_step/sec: 31.4956\n",
      "INFO:tensorflow:loss = 0.13405277, step = 301 (3.175 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.0454\n",
      "INFO:tensorflow:loss = 0.069856234, step = 401 (3.026 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.8046\n",
      "INFO:tensorflow:loss = 0.08389125, step = 501 (2.958 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9307\n",
      "INFO:tensorflow:loss = 0.010030745, step = 601 (2.947 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1624\n",
      "INFO:tensorflow:loss = 0.036355894, step = 701 (2.927 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0461\n",
      "INFO:tensorflow:loss = 0.027569551, step = 801 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 32.7963\n",
      "INFO:tensorflow:loss = 0.014463619, step = 901 (3.050 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0114\n",
      "INFO:tensorflow:loss = 0.078560084, step = 1001 (2.940 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0925\n",
      "INFO:tensorflow:loss = 0.08021919, step = 1101 (2.932 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9192\n",
      "INFO:tensorflow:loss = 0.022794157, step = 1201 (2.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9652\n",
      "INFO:tensorflow:loss = 0.0028004805, step = 1301 (2.944 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0693\n",
      "INFO:tensorflow:loss = 0.01804494, step = 1401 (2.935 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.4319\n",
      "INFO:tensorflow:loss = 0.028760552, step = 1501 (2.991 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9422\n",
      "INFO:tensorflow:loss = 0.05222799, step = 1601 (2.946 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9999\n",
      "INFO:tensorflow:loss = 0.01154986, step = 1701 (2.941 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.0577\n",
      "INFO:tensorflow:loss = 0.015789155, step = 1801 (2.936 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.1274\n",
      "INFO:tensorflow:loss = 0.013321694, step = 1901 (2.930 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.1659\n",
      "INFO:tensorflow:loss = 0.011996182, step = 2001 (3.015 sec)\n",
      "INFO:tensorflow:global_step/sec: 34.2442\n",
      "INFO:tensorflow:loss = 0.025972545, step = 2101 (2.920 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.9077\n",
      "INFO:tensorflow:loss = 0.016873557, step = 2201 (2.949 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.221\n",
      "INFO:tensorflow:loss = 0.072110005, step = 2301 (3.010 sec)\n",
      "INFO:tensorflow:global_step/sec: 33.21\n",
      "INFO:tensorflow:loss = 0.0070967535, step = 2401 (3.011 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2500 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-10-20:32:09\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-2500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving candidate 'simple_cnn' dict for global step 2500: accuracy/adanet/adanet_weighted_ensemble = 0.9868, accuracy/adanet/subnetwork = 0.9868, accuracy/adanet/uniform_average_ensemble = 0.9868, architecture/adanet/ensembles = b\"\\n_\\n:adanet/iteration_0/ensemble_simple_cnn/architecture/adanetB\\x17\\x08\\x07\\x12\\x00B\\x11| b'simple_cnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.04272877, average_loss/adanet/subnetwork = 0.04272877, average_loss/adanet/uniform_average_ensemble = 0.04272877, loss/adanet/adanet_weighted_ensemble = 0.042525534, loss/adanet/subnetwork = 0.042525534, loss/adanet/uniform_average_ensemble = 0.042525534\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-10-20:32:13\n",
      "INFO:tensorflow:Saving dict for global step 2500: accuracy = 0.9868, accuracy/adanet/adanet_weighted_ensemble = 0.9868, accuracy/adanet/subnetwork = 0.9868, accuracy/adanet/uniform_average_ensemble = 0.9868, average_loss = 0.04272877, average_loss/adanet/adanet_weighted_ensemble = 0.04272877, average_loss/adanet/subnetwork = 0.04272877, average_loss/adanet/uniform_average_ensemble = 0.04272877, global_step = 2500, loss = 0.042525534, loss/adanet/adanet_weighted_ensemble = 0.042525534, loss/adanet/subnetwork = 0.042525534, loss/adanet/uniform_average_ensemble = 0.042525534\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2500: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-2500\n",
      "INFO:tensorflow:Loss for final step: 0.02642705.\n",
      "INFO:tensorflow:As the only candidate, 'simple_cnn' is moving onto the next iteration.\n",
      "INFO:tensorflow:Starting metric logging for iteration 0\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-2500\n",
      "WARNING:tensorflow:`tf.train.start_queue_runners()` was called when no queue runners were defined. You can safely remove the call to this deprecated function.\n",
      "INFO:tensorflow:Encountered end of input during report materialization\n",
      "INFO:tensorflow:Materialized subnetwork_reports.\n",
      "INFO:tensorflow:Wrote IterationReport for iteration 0 to C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\report\\iteration_reports.tfrecord\n",
      "INFO:tensorflow:Finished saving subnetwork reports for iteration 0\n",
      "INFO:tensorflow:Freezing best ensemble to C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\frozen/ensemble-0.meta\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-2500\n",
      "INFO:tensorflow:Importing frozen ensemble from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\frozen/ensemble-0.meta with features: ['images'].\n",
      "INFO:tensorflow:Overwriting checkpoint with new graph for iteration 1 to C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-2500\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-2500\n",
      "WARNING:tensorflow:`tf.train.start_queue_runners()` was called when no queue runners were defined. You can safely remove the call to this deprecated function.\n",
      "INFO:tensorflow:Finished training Adanet iteration 0\n",
      "INFO:tensorflow:Beginning training AdaNet iteration 1\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Importing frozen ensemble from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\frozen/ensemble-0.meta with features: ['images'].\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\increment.ckpt-1\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2500 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.02847917, step = 2501\n",
      "INFO:tensorflow:global_step/sec: 23.8725\n",
      "INFO:tensorflow:loss = 0.004389563, step = 2601 (4.190 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.4853\n",
      "INFO:tensorflow:loss = 0.0059942594, step = 2701 (3.776 sec)\n",
      "INFO:tensorflow:global_step/sec: 25.4221\n",
      "INFO:tensorflow:loss = 0.080152735, step = 2801 (3.975 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.6401\n",
      "INFO:tensorflow:loss = 0.008667019, step = 2901 (3.713 sec)\n",
      "INFO:tensorflow:global_step/sec: 25.9765\n",
      "INFO:tensorflow:loss = 0.042215403, step = 3001 (3.850 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.4994\n",
      "INFO:tensorflow:loss = 0.0013359454, step = 3101 (3.774 sec)\n",
      "INFO:tensorflow:global_step/sec: 27.4066\n",
      "INFO:tensorflow:loss = 0.0086673405, step = 3201 (3.649 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.598\n",
      "INFO:tensorflow:loss = 0.004777601, step = 3301 (3.760 sec)\n",
      "INFO:tensorflow:global_step/sec: 24.7419\n",
      "INFO:tensorflow:loss = 0.0063926866, step = 3401 (4.042 sec)\n",
      "INFO:tensorflow:global_step/sec: 27.2555\n",
      "INFO:tensorflow:loss = 0.020389045, step = 3501 (3.668 sec)\n",
      "INFO:tensorflow:global_step/sec: 27.5753\n",
      "INFO:tensorflow:loss = 0.023859661, step = 3601 (3.626 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.3601\n",
      "INFO:tensorflow:loss = 0.0048056315, step = 3701 (3.795 sec)\n",
      "INFO:tensorflow:global_step/sec: 25.7127\n",
      "INFO:tensorflow:loss = 0.0020669564, step = 3801 (3.888 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.8676\n",
      "INFO:tensorflow:loss = 0.010222295, step = 3901 (3.723 sec)\n",
      "INFO:tensorflow:global_step/sec: 26.4409\n",
      "INFO:tensorflow:loss = 0.0240863, step = 4001 (3.781 sec)\n",
      "INFO:tensorflow:global_step/sec: 27.9045\n",
      "INFO:tensorflow:loss = 0.025914751, step = 4101 (3.584 sec)\n",
      "INFO:tensorflow:global_step/sec: 27.4344\n",
      "INFO:tensorflow:loss = 0.01542864, step = 4201 (3.645 sec)\n",
      "INFO:tensorflow:global_step/sec: 28.027\n",
      "INFO:tensorflow:loss = 0.013081177, step = 4301 (3.568 sec)\n",
      "INFO:tensorflow:global_step/sec: 27.8596\n",
      "INFO:tensorflow:loss = 0.01050403, step = 4401 (3.590 sec)\n",
      "INFO:tensorflow:global_step/sec: 25.493\n",
      "INFO:tensorflow:loss = 0.011618882, step = 4501 (3.922 sec)\n",
      "INFO:tensorflow:global_step/sec: 25.7558\n",
      "INFO:tensorflow:loss = 0.016643602, step = 4601 (3.883 sec)\n",
      "INFO:tensorflow:global_step/sec: 25.4344\n",
      "INFO:tensorflow:loss = 0.014137244, step = 4701 (3.933 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.7484\n",
      "INFO:tensorflow:loss = 0.058484245, step = 4801 (4.211 sec)\n",
      "INFO:tensorflow:global_step/sec: 23.342\n",
      "INFO:tensorflow:loss = 0.006717515, step = 4901 (4.284 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5000 into C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Importing frozen ensemble from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\frozen/ensemble-0.meta with features: ['images'].\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2018-11-10-20:34:16\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-5000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving candidate 'previous_ensemble' dict for global step 5000: accuracy/adanet/adanet_weighted_ensemble = 0.9868, accuracy/adanet/subnetwork = 0.9868, accuracy/adanet/uniform_average_ensemble = 0.9868, architecture/adanet/ensembles = b\"\\nQ\\n,adanet/previous_ensemble/architecture/adanetB\\x17\\x08\\x07\\x12\\x00B\\x11| b'simple_cnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.04272877, average_loss/adanet/subnetwork = 0.04272877, average_loss/adanet/uniform_average_ensemble = 0.04272877, loss/adanet/adanet_weighted_ensemble = 0.042525534, loss/adanet/subnetwork = 0.042525534, loss/adanet/uniform_average_ensemble = 0.042525534\n",
      "INFO:tensorflow:Saving candidate 'simple_cnn' dict for global step 5000: accuracy/adanet/adanet_weighted_ensemble = 0.9869, accuracy/adanet/subnetwork = 0.9856, accuracy/adanet/uniform_average_ensemble = 0.9869, architecture/adanet/ensembles = b\"\\no\\n:adanet/iteration_1/ensemble_simple_cnn/architecture/adanetB'\\x08\\x07\\x12\\x00B!| b'simple_cnn' | b'simple_cnn' |J\\x08\\n\\x06\\n\\x04text\", average_loss/adanet/adanet_weighted_ensemble = 0.03953884, average_loss/adanet/subnetwork = 0.04450064, average_loss/adanet/uniform_average_ensemble = 0.03953884, loss/adanet/adanet_weighted_ensemble = 0.03935165, loss/adanet/subnetwork = 0.044292618, loss/adanet/uniform_average_ensemble = 0.03935165\n",
      "INFO:tensorflow:Finished evaluation at 2018-11-10-20:34:22\n",
      "INFO:tensorflow:Saving dict for global step 5000: accuracy = 0.9869, accuracy/adanet/adanet_weighted_ensemble = 0.9869, accuracy/adanet/subnetwork = 0.9856, accuracy/adanet/uniform_average_ensemble = 0.9869, average_loss = 0.03953884, average_loss/adanet/adanet_weighted_ensemble = 0.03953884, average_loss/adanet/subnetwork = 0.04450064, average_loss/adanet/uniform_average_ensemble = 0.03953884, global_step = 5000, loss = 0.03935165, loss/adanet/adanet_weighted_ensemble = 0.03935165, loss/adanet/subnetwork = 0.044292618, loss/adanet/uniform_average_ensemble = 0.03935165\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5000: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-5000\n",
      "INFO:tensorflow:Loss for final step: 0.02186182.\n",
      "Accuracy: 0.9869\n",
      "Loss: 0.03953884\n"
     ]
    }
   ],
   "source": [
    "LEARNING_RATE = 0.05  #@param {type:\"number\"}\n",
    "TRAIN_STEPS = 5000  #@param {type:\"integer\"}\n",
    "BATCH_SIZE = 64  #@param {type:\"integer\"}\n",
    "ADANET_ITERATIONS = 2  #@param {type:\"integer\"}\n",
    "\n",
    "max_iteration_steps = TRAIN_STEPS // ADANET_ITERATIONS\n",
    "estimator = adanet.Estimator(\n",
    "    head=head,\n",
    "    subnetwork_generator=SimpleCNNGenerator(\n",
    "        learning_rate=LEARNING_RATE,\n",
    "        max_iteration_steps=max_iteration_steps,\n",
    "        seed=RANDOM_SEED),\n",
    "    max_iteration_steps=max_iteration_steps,\n",
    "    evaluator=adanet.Evaluator(\n",
    "        input_fn=input_fn(\"train\", training=False, batch_size=BATCH_SIZE),\n",
    "        steps=None),\n",
    "    report_materializer=adanet.ReportMaterializer(\n",
    "        input_fn=input_fn(\"train\", training=False, batch_size=BATCH_SIZE),\n",
    "        steps=None),\n",
    "    adanet_loss_decay=.99,\n",
    "    config=config)\n",
    "\n",
    "results, _ = tf.estimator.train_and_evaluate(\n",
    "    estimator,\n",
    "    train_spec=tf.estimator.TrainSpec(\n",
    "        input_fn=input_fn(\"train\", training=True, batch_size=BATCH_SIZE),\n",
    "        max_steps=TRAIN_STEPS),\n",
    "    eval_spec=tf.estimator.EvalSpec(\n",
    "        input_fn=input_fn(\"test\", training=False, batch_size=BATCH_SIZE),\n",
    "        steps=None))\n",
    "print(\"Accuracy:\", results[\"accuracy\"])\n",
    "print(\"Loss:\", results[\"average_loss\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving dict for global step 5000: accuracy = 0.9869, accuracy/adanet/adanet_weighted_ensemble = 0.9869, accuracy/adanet/subnetwork = 0.9856, accuracy/adanet/uniform_average_ensemble = 0.9869, average_loss = 0.03953884, average_loss/adanet/adanet_weighted_ensemble = 0.03953884, average_loss/adanet/subnetwork = 0.04450064, average_loss/adanet/uniform_average_ensemble = 0.03953884, global_step = 5000, loss = 0.03935165, loss/adanet/adanet_weighted_ensemble = 0.03935165, loss/adanet/subnetwork = 0.044292618, loss/adanet/uniform_average_ensemble = 0.03935165\n",
    "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5000: C:\\Users\\dell\\AppData\\Local\\Temp\\tmp3rlexm4i\\model.ckpt-5000\n",
    "INFO:tensorflow:Loss for final step: 0.02186182.\n",
    "Accuracy: 0.9869\n",
    "Loss: 0.03953884"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ve MNIST icin %98.69 basarim elde etmis olduk."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}