03-Factorization-Machines-Movielens.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Movie recommendation on Amazon SageMaker with Factorization Machines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download ml-100k dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2018-08-29 04:07:00--  http://files.grouplens.org/datasets/movielens/ml-100k.zip\n",
      "Resolving files.grouplens.org (files.grouplens.org)... 128.101.34.235\n",
      "Connecting to files.grouplens.org (files.grouplens.org)|128.101.34.235|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 4924029 (4.7M) [application/zip]\n",
      "Saving to: ‘ml-100k.zip’\n",
      "\n",
      "ml-100k.zip         100%[===================>]   4.70M  21.1MB/s    in 0.2s    \n",
      "\n",
      "2018-08-29 04:07:01 (21.1 MB/s) - ‘ml-100k.zip’ saved [4924029/4924029]\n",
      "\n",
      "Archive:  ml-100k.zip\n",
      "   creating: ml-100k/\n",
      "  inflating: ml-100k/allbut.pl       \n",
      "  inflating: ml-100k/mku.sh          \n",
      "  inflating: ml-100k/README          \n",
      "  inflating: ml-100k/u.data          \n",
      "  inflating: ml-100k/u.genre         \n",
      "  inflating: ml-100k/u.info          \n",
      "  inflating: ml-100k/u.item          \n",
      "  inflating: ml-100k/u.occupation    \n",
      "  inflating: ml-100k/u.user          \n",
      "  inflating: ml-100k/u1.base         \n",
      "  inflating: ml-100k/u1.test         \n",
      "  inflating: ml-100k/u2.base         \n",
      "  inflating: ml-100k/u2.test         \n",
      "  inflating: ml-100k/u3.base         \n",
      "  inflating: ml-100k/u3.test         \n",
      "  inflating: ml-100k/u4.base         \n",
      "  inflating: ml-100k/u4.test         \n",
      "  inflating: ml-100k/u5.base         \n",
      "  inflating: ml-100k/u5.test         \n",
      "  inflating: ml-100k/ua.base         \n",
      "  inflating: ml-100k/ua.test         \n",
      "  inflating: ml-100k/ub.base         \n",
      "  inflating: ml-100k/ub.test         \n"
     ]
    }
   ],
   "source": [
    "!wget http://files.grouplens.org/datasets/movielens/ml-100k.zip\n",
    "!unzip -o ml-100k.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/SageMaker/pilho-lab/ml-100k\n",
      "343\t25\t2\t876402814\n",
      "659\t212\t4\t891387227\n",
      "858\t515\t4\t880932911\n",
      "387\t196\t2\t886484012\n",
      "854\t762\t2\t882812905\n",
      "421\t466\t4\t892241459\n",
      "436\t895\t4\t887768717\n",
      "605\t462\t5\t881016176\n",
      "346\t276\t1\t874950029\n",
      "40\t294\t4\t889041671\n"
     ]
    }
   ],
   "source": [
    "%cd ml-100k\n",
    "!shuf ua.base -o ua.base.shuffled\n",
    "!head -10 ua.base.shuffled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\t20\t4\t887431883\r\n",
      "1\t33\t4\t878542699\r\n",
      "1\t61\t4\t878542420\r\n",
      "1\t117\t3\t874965739\r\n",
      "1\t155\t2\t878542201\r\n",
      "1\t160\t4\t875072547\r\n",
      "1\t171\t5\t889751711\r\n",
      "1\t189\t3\t888732928\r\n",
      "1\t202\t5\t875072442\r\n",
      "1\t265\t4\t878542441\r\n"
     ]
    }
   ],
   "source": [
    "!head -10 ua.test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "import sagemaker.amazon.common as smac\n",
    "from sagemaker import get_execution_role\n",
    "from sagemaker.predictor import json_deserializer\n",
    "\n",
    "import boto3, csv, io, json\n",
    "import numpy as np\n",
    "from scipy.sparse import lil_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build training set and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "nbUsers=943\n",
    "nbMovies=1682\n",
    "nbFeatures=nbUsers+nbMovies\n",
    "\n",
    "nbRatingsTrain=90570\n",
    "nbRatingsTest=9430"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For each user, build a list of rated movies.\n",
    "# We'd need this to add random negative samples.\n",
    "moviesByUser = {}\n",
    "for userId in range(nbUsers):\n",
    "    moviesByUser[str(userId)]=[]\n",
    " \n",
    "with open('ua.base.shuffled','r') as f:\n",
    "    samples=csv.reader(f,delimiter='\\t')\n",
    "    for userId,movieId,rating,timestamp in samples:\n",
    "        moviesByUser[str(int(userId)-1)].append(int(movieId)-1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadDataset(filename, lines, columns):\n",
    "    # Features are one-hot encoded in a sparse matrix\n",
    "    X = lil_matrix((lines, columns)).astype('float32')\n",
    "    # Labels are stored in a vector\n",
    "    Y = []\n",
    "    line=0\n",
    "    with open(filename,'r') as f:\n",
    "        samples=csv.reader(f,delimiter='\\t')\n",
    "        for userId,movieId,rating,timestamp in samples:\n",
    "            X[line,int(userId)-1] = 1\n",
    "            X[line,int(nbUsers)+int(movieId)-1] = 1\n",
    "            if int(rating) >= 4:\n",
    "                Y.append(1)\n",
    "            else:\n",
    "                Y.append(0)\n",
    "            line=line+1\n",
    "            \n",
    "    Y=np.array(Y).astype('float32')\n",
    "    return X,Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, Y_train = loadDataset('ua.base.shuffled', nbRatingsTrain, nbFeatures)\n",
    "X_test, Y_test = loadDataset('ua.test',nbRatingsTest,nbFeatures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 591)\t1.0\n",
      "  (0, 1403)\t1.0\n"
     ]
    }
   ],
   "source": [
    "print(X_train[1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(90570, 2625)\n",
      "(90570,)\n",
      "Training labels: 49906 zeros, 40664 ones\n",
      "(9430, 2625)\n",
      "(9430,)\n",
      "Test labels: 5469 zeros, 3961 ones\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(Y_train.shape)\n",
    "assert X_train.shape == (nbRatingsTrain, nbFeatures)\n",
    "assert Y_train.shape == (nbRatingsTrain, )\n",
    "zero_labels = np.count_nonzero(Y_train)\n",
    "print(\"Training labels: %d zeros, %d ones\" % (zero_labels, nbRatingsTrain-zero_labels))\n",
    "\n",
    "print(X_test.shape)\n",
    "print(Y_test.shape)\n",
    "assert X_test.shape  == (nbRatingsTest, nbFeatures)\n",
    "assert Y_test.shape  == (nbRatingsTest, )\n",
    "zero_labels = np.count_nonzero(Y_test)\n",
    "print(\"Test labels: %d zeros, %d ones\" % (zero_labels, nbRatingsTest-zero_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert to protobuf and save to S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket = 'pilho-sagemaker-ai-workshop'\n",
    "prefix = 'sagemaker/fm-movielens'\n",
    "\n",
    "train_key      = 'train.protobuf'\n",
    "train_prefix   = '{}/{}'.format(prefix, 'train3')\n",
    "\n",
    "test_key       = 'test.protobuf'\n",
    "test_prefix    = '{}/{}'.format(prefix, 'test3')\n",
    "\n",
    "output_prefix  = 's3://{}/{}/output'.format(bucket, prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://pilho-sagemaker-ai-workshop/sagemaker/fm-movielens/train3/train.protobuf\n",
      "s3://pilho-sagemaker-ai-workshop/sagemaker/fm-movielens/test3/test.protobuf\n",
      "Output: s3://pilho-sagemaker-ai-workshop/sagemaker/fm-movielens/output\n"
     ]
    }
   ],
   "source": [
    "def writeDatasetToProtobuf(X, Y, bucket, prefix, key):\n",
    "    buf = io.BytesIO()\n",
    "    smac.write_spmatrix_to_sparse_tensor(buf, X, Y)\n",
    "    buf.seek(0)\n",
    "    obj = '{}/{}'.format(prefix, key)\n",
    "    boto3.resource('s3').Bucket(bucket).Object(obj).upload_fileobj(buf)\n",
    "    return 's3://{}/{}'.format(bucket,obj)\n",
    "    \n",
    "train_data = writeDatasetToProtobuf(X_train, Y_train, bucket, train_prefix, train_key)    \n",
    "test_data  = writeDatasetToProtobuf(X_test, Y_test, bucket, test_prefix, test_key)    \n",
    "  \n",
    "print(train_data)\n",
    "print(test_data)\n",
    "print('Output: {}'.format(output_prefix))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run training job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "containers = {'us-west-2': '174872318107.dkr.ecr.us-west-2.amazonaws.com/factorization-machines:latest',\n",
    "              'us-east-1': '382416733822.dkr.ecr.us-east-1.amazonaws.com/factorization-machines:latest',\n",
    "              'us-east-2': '404615174143.dkr.ecr.us-east-2.amazonaws.com/factorization-machines:latest',\n",
    "              'eu-west-1': '438346466558.dkr.ecr.eu-west-1.amazonaws.com/factorization-machines:latest'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating training-job with name: factorization-machines-2018-08-29-04-07-20-066\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".....................\n",
      "\u001b[31mDocker entrypoint called with argument(s): train\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] Reading default configuration from /opt/amazon/lib/python2.7/site-packages/algorithm/resources/default-conf.json: {u'factors_lr': u'0.0001', u'linear_init_sigma': u'0.01', u'epochs': 1, u'_wd': u'1.0', u'_num_kv_servers': u'auto', u'use_bias': u'true', u'factors_init_sigma': u'0.001', u'_log_level': u'info', u'bias_init_method': u'normal', u'linear_init_method': u'normal', u'linear_lr': u'0.001', u'factors_init_method': u'normal', u'_tuning_objective_metric': u'', u'bias_wd': u'0.01', u'use_linear': u'true', u'bias_lr': u'0.1', u'mini_batch_size': u'1000', u'_use_full_symbolic': u'true', u'batch_metrics_publish_interval': u'500', u'bias_init_sigma': u'0.01', u'_num_gpus': u'auto', u'_data_format': u'record', u'factors_wd': u'0.00001', u'linear_wd': u'0.001', u'_kvstore': u'auto', u'_learning_rate': u'1.0', u'_optimizer': u'adam'}\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] Reading provided configuration from /opt/ml/input/config/hyperparameters.json: {u'epochs': u'100', u'feature_dim': u'2625', u'mini_batch_size': u'1000', u'predictor_type': u'binary_classifier', u'num_factors': u'64'}\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] Final configuration: {u'factors_lr': u'0.0001', u'linear_init_sigma': u'0.01', u'epochs': u'100', u'feature_dim': u'2625', u'num_factors': u'64', u'_wd': u'1.0', u'_num_kv_servers': u'auto', u'use_bias': u'true', u'factors_init_sigma': u'0.001', u'_log_level': u'info', u'bias_init_method': u'normal', u'linear_init_method': u'normal', u'linear_lr': u'0.001', u'factors_init_method': u'normal', u'_tuning_objective_metric': u'', u'bias_wd': u'0.01', u'use_linear': u'true', u'bias_lr': u'0.1', u'mini_batch_size': u'1000', u'_use_full_symbolic': u'true', u'batch_metrics_publish_interval': u'500', u'predictor_type': u'binary_classifier', u'bias_init_sigma': u'0.01', u'_num_gpus': u'auto', u'_data_format': u'record', u'factors_wd': u'0.00001', u'linear_wd': u'0.001', u'_kvstore': u'auto', u'_learning_rate': u'1.0', u'_optimizer': u'adam'}\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 WARNING 139937213605696] Loggers have already been setup.\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] Using default worker.\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] nvidia-smi took: 0.0252070426941 secs to identify 0 gpus\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] Number of GPUs being used: 0\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] [Sparse network] Building a sparse network.\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] Create Store: local\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"initialize.time\": {\"count\": 1, \"max\": 35.96806526184082, \"sum\": 35.96806526184082, \"min\": 35.96806526184082}}, \"EndTime\": 1535515843.276798, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515843.216106}\n",
      "\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 0, \"sum\": 0.0, \"min\": 0}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}, \"Total Records Seen\": {\"count\": 1, \"max\": 1000, \"sum\": 1000.0, \"min\": 1000}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 1000, \"sum\": 1000.0, \"min\": 1000}, \"Reset Count\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}}, \"EndTime\": 1535515843.276978, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"init_train_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515843.276944}\n",
      "\u001b[0m\n",
      "\u001b[31m[04:10:43] /opt/brazil-pkg-cache/packages/AIAlgorithmsMXNet/AIAlgorithmsMXNet-1.1.x.200686.0/RHEL5_64/generic-flavor/src/src/kvstore/./kvstore_local.h:280: Warning: non-default weights detected during kvstore pull. This call has been ignored. Please make sure to use row_sparse_pull with row_ids.\u001b[0m\n",
      "\u001b[31m[04:10:43] /opt/brazil-pkg-cache/packages/AIAlgorithmsMXNet/AIAlgorithmsMXNet-1.1.x.200686.0/RHEL5_64/generic-flavor/src/src/kvstore/./kvstore_local.h:280: Warning: non-default weights detected during kvstore pull. This call has been ignored. Please make sure to use row_sparse_pull with row_ids.\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=0, batch=0 train binary_classification_accuracy <score>=0.56\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=0, batch=0 train binary_classification_cross_entropy <loss>=0.692126342773\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=0, batch=0 train binary_f_1.000 <score>=0.707057256991\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=0, train binary_classification_accuracy <score>=0.550659340659\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=0, train binary_classification_cross_entropy <loss>=0.683741668366\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=0, train binary_f_1.000 <score>=0.709054944429\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"epochs\": {\"count\": 1, \"max\": 100, \"sum\": 100.0, \"min\": 100}, \"update.time\": {\"count\": 1, \"max\": 883.6288452148438, \"sum\": 883.6288452148438, \"min\": 883.6288452148438}}, \"EndTime\": 1535515844.160841, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515843.276883}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #progress_metric: host=algo-1, completed 1 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 92, \"sum\": 92.0, \"min\": 92}, \"Total Records Seen\": {\"count\": 1, \"max\": 91570, \"sum\": 91570.0, \"min\": 91570}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 2, \"sum\": 2.0, \"min\": 2}}, \"EndTime\": 1535515844.161035, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 0}, \"StartTime\": 1535515843.277184}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=102458.225647 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=1, batch=0 train binary_classification_accuracy <score>=0.665\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=1, batch=0 train binary_classification_cross_entropy <loss>=0.679037902832\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=1, batch=0 train binary_f_1.000 <score>=0.756008739985\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=1, train binary_classification_accuracy <score>=0.565175824176\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=1, train binary_classification_cross_entropy <loss>=0.673626895443\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=1, train binary_f_1.000 <score>=0.71407822763\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 769.3071365356445, \"sum\": 769.3071365356445, \"min\": 769.3071365356445}}, \"EndTime\": 1535515844.930661, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515844.160917}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #progress_metric: host=algo-1, completed 2 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 183, \"sum\": 183.0, \"min\": 183}, \"Total Records Seen\": {\"count\": 1, \"max\": 182140, \"sum\": 182140.0, \"min\": 182140}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 3, \"sum\": 3.0, \"min\": 3}}, \"EndTime\": 1535515844.930885, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 1}, \"StartTime\": 1535515844.161322}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=117670.599142 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=2, batch=0 train binary_classification_accuracy <score>=0.72\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=2, batch=0 train binary_classification_cross_entropy <loss>=0.66946295166\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=2, batch=0 train binary_f_1.000 <score>=0.76897689769\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=2, train binary_classification_accuracy <score>=0.600835164835\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=2, train binary_classification_cross_entropy <loss>=0.664315983028\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=2, train binary_f_1.000 <score>=0.727289107781\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 759.0601444244385, \"sum\": 759.0601444244385, \"min\": 759.0601444244385}}, \"EndTime\": 1535515845.690339, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515844.930742}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #progress_metric: host=algo-1, completed 3 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 274, \"sum\": 274.0, \"min\": 274}, \"Total Records Seen\": {\"count\": 1, \"max\": 272710, \"sum\": 272710.0, \"min\": 272710}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 4, \"sum\": 4.0, \"min\": 4}}, \"EndTime\": 1535515845.690552, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 2}, \"StartTime\": 1535515844.931245}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119258.749033 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=3, batch=0 train binary_classification_accuracy <score>=0.717\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=3, batch=0 train binary_classification_cross_entropy <loss>=0.66024432373\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=3, batch=0 train binary_f_1.000 <score>=0.757081545064\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=3, train binary_classification_accuracy <score>=0.634043956044\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=3, train binary_classification_cross_entropy <loss>=0.655583987058\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=3, train binary_f_1.000 <score>=0.739673556174\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 774.5270729064941, \"sum\": 774.5270729064941, \"min\": 774.5270729064941}}, \"EndTime\": 1535515846.465406, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515845.690405}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #progress_metric: host=algo-1, completed 4 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 365, \"sum\": 365.0, \"min\": 365}, \"Total Records Seen\": {\"count\": 1, \"max\": 363280, \"sum\": 363280.0, \"min\": 363280}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 5, \"sum\": 5.0, \"min\": 5}}, \"EndTime\": 1535515846.465597, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 3}, \"StartTime\": 1535515845.690847}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=116884.134743 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=4, batch=0 train binary_classification_accuracy <score>=0.714\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=4, batch=0 train binary_classification_cross_entropy <loss>=0.651512817383\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=4, batch=0 train binary_f_1.000 <score>=0.749562171629\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=4, train binary_classification_accuracy <score>=0.656978021978\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=4, train binary_classification_cross_entropy <loss>=0.647403231107\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=4, train binary_f_1.000 <score>=0.747767766959\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 695.1780319213867, \"sum\": 695.1780319213867, \"min\": 695.1780319213867}}, \"EndTime\": 1535515847.161187, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515846.465467}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #progress_metric: host=algo-1, completed 5 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 456, \"sum\": 456.0, \"min\": 456}, \"Total Records Seen\": {\"count\": 1, \"max\": 453850, \"sum\": 453850.0, \"min\": 453850}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 6, \"sum\": 6.0, \"min\": 6}}, \"EndTime\": 1535515847.161393, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 4}, \"StartTime\": 1535515846.465967}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=130209.757513 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=5, batch=0 train binary_classification_accuracy <score>=0.717\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=5, batch=0 train binary_classification_cross_entropy <loss>=0.643337219238\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=5, batch=0 train binary_f_1.000 <score>=0.748890860692\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=5, train binary_classification_accuracy <score>=0.673582417582\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=5, train binary_classification_cross_entropy <loss>=0.639793387948\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=5, train binary_f_1.000 <score>=0.753481501154\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 772.3510265350342, \"sum\": 772.3510265350342, \"min\": 772.3510265350342}}, \"EndTime\": 1535515847.934051, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515847.161255}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #progress_metric: host=algo-1, completed 6 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 547, \"sum\": 547.0, \"min\": 547}, \"Total Records Seen\": {\"count\": 1, \"max\": 544420, \"sum\": 544420.0, \"min\": 544420}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 7, \"sum\": 7.0, \"min\": 7}}, \"EndTime\": 1535515847.934259, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 5}, \"StartTime\": 1535515847.161666}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=117210.5531 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=6, batch=0 train binary_classification_accuracy <score>=0.72\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=6, batch=0 train binary_classification_cross_entropy <loss>=0.635737182617\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=6, batch=0 train binary_f_1.000 <score>=0.749103942652\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=6, train binary_classification_accuracy <score>=0.685967032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=6, train binary_classification_cross_entropy <loss>=0.632750747848\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=6, train binary_f_1.000 <score>=0.757778926758\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 712.2139930725098, \"sum\": 712.2139930725098, \"min\": 712.2139930725098}}, \"EndTime\": 1535515848.646776, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515847.934119}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #progress_metric: host=algo-1, completed 7 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 638, \"sum\": 638.0, \"min\": 638}, \"Total Records Seen\": {\"count\": 1, \"max\": 634990, \"sum\": 634990.0, \"min\": 634990}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 8, \"sum\": 8.0, \"min\": 8}}, \"EndTime\": 1535515848.646995, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 6}, \"StartTime\": 1535515847.934532}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127100.801621 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=7, batch=0 train binary_classification_accuracy <score>=0.719\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=7, batch=0 train binary_classification_cross_entropy <loss>=0.628700378418\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=7, batch=0 train binary_f_1.000 <score>=0.746160794941\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=7, train binary_classification_accuracy <score>=0.695472527473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=7, train binary_classification_cross_entropy <loss>=0.626251745203\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=7, train binary_f_1.000 <score>=0.760876693416\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 744.0760135650635, \"sum\": 744.0760135650635, \"min\": 744.0760135650635}}, \"EndTime\": 1535515849.391431, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515848.646841}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #progress_metric: host=algo-1, completed 8 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 729, \"sum\": 729.0, \"min\": 729}, \"Total Records Seen\": {\"count\": 1, \"max\": 725560, \"sum\": 725560.0, \"min\": 725560}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 9, \"sum\": 9.0, \"min\": 9}}, \"EndTime\": 1535515849.391644, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 7}, \"StartTime\": 1535515848.647324}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121665.024712 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=8, batch=0 train binary_classification_accuracy <score>=0.725\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=8, batch=0 train binary_classification_cross_entropy <loss>=0.622197631836\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=8, batch=0 train binary_f_1.000 <score>=0.750679963735\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=8, train binary_classification_accuracy <score>=0.70167032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=8, train binary_classification_cross_entropy <loss>=0.620262380076\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=8, train binary_f_1.000 <score>=0.762426491179\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 730.7350635528564, \"sum\": 730.7350635528564, \"min\": 730.7350635528564}}, \"EndTime\": 1535515850.122713, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515849.391502}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #progress_metric: host=algo-1, completed 9 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 820, \"sum\": 820.0, \"min\": 820}, \"Total Records Seen\": {\"count\": 1, \"max\": 816130, \"sum\": 816130.0, \"min\": 816130}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 10, \"sum\": 10.0, \"min\": 10}}, \"EndTime\": 1535515850.122968, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 8}, \"StartTime\": 1535515849.391941}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123874.213401 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=9, batch=0 train binary_classification_accuracy <score>=0.729\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=9, batch=0 train binary_classification_cross_entropy <loss>=0.616192504883\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=9, batch=0 train binary_f_1.000 <score>=0.753860127157\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=9, train binary_classification_accuracy <score>=0.707340659341\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=9, train binary_classification_cross_entropy <loss>=0.614744224465\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=9, train binary_f_1.000 <score>=0.764297725462\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 758.2180500030518, \"sum\": 758.2180500030518, \"min\": 758.2180500030518}}, \"EndTime\": 1535515850.881719, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515850.122784}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #progress_metric: host=algo-1, completed 10 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 911, \"sum\": 911.0, \"min\": 911}, \"Total Records Seen\": {\"count\": 1, \"max\": 906700, \"sum\": 906700.0, \"min\": 906700}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 11, \"sum\": 11.0, \"min\": 11}}, \"EndTime\": 1535515850.88194, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 9}, \"StartTime\": 1535515850.123472}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119394.586258 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=10, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=10, batch=0 train binary_classification_cross_entropy <loss>=0.610645629883\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=10, batch=0 train binary_f_1.000 <score>=0.756119673617\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=10, train binary_classification_accuracy <score>=0.711406593407\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=10, train binary_classification_cross_entropy <loss>=0.609657893925\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=10, train binary_f_1.000 <score>=0.76527894464\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 727.1180152893066, \"sum\": 727.1180152893066, \"min\": 727.1180152893066}}, \"EndTime\": 1535515851.609428, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515850.881789}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #progress_metric: host=algo-1, completed 11 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1002, \"sum\": 1002.0, \"min\": 1002}, \"Total Records Seen\": {\"count\": 1, \"max\": 997270, \"sum\": 997270.0, \"min\": 997270}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 12, \"sum\": 12.0, \"min\": 12}}, \"EndTime\": 1535515851.609672, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 10}, \"StartTime\": 1535515850.882278}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124492.281552 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=11, batch=0 train binary_classification_accuracy <score>=0.729\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=11, batch=0 train binary_classification_cross_entropy <loss>=0.605518310547\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=11, batch=0 train binary_f_1.000 <score>=0.75430643699\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=11, train binary_classification_accuracy <score>=0.714505494505\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=11, train binary_classification_cross_entropy <loss>=0.60496499399\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=11, train binary_f_1.000 <score>=0.76595016306\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 779.9901962280273, \"sum\": 779.9901962280273, \"min\": 779.9901962280273}}, \"EndTime\": 1535515852.391443, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515851.609498}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #progress_metric: host=algo-1, completed 12 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1093, \"sum\": 1093.0, \"min\": 1093}, \"Total Records Seen\": {\"count\": 1, \"max\": 1087840, \"sum\": 1087840.0, \"min\": 1087840}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 13, \"sum\": 13.0, \"min\": 13}}, \"EndTime\": 1535515852.391655, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 11}, \"StartTime\": 1535515851.611421}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=116061.536408 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=12, batch=0 train binary_classification_accuracy <score>=0.729\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=12, batch=0 train binary_classification_cross_entropy <loss>=0.60077331543\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=12, batch=0 train binary_f_1.000 <score>=0.75430643699\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=12, train binary_classification_accuracy <score>=0.71778021978\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=12, train binary_classification_cross_entropy <loss>=0.60062918728\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=12, train binary_f_1.000 <score>=0.767001741907\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 749.2771148681641, \"sum\": 749.2771148681641, \"min\": 749.2771148681641}}, \"EndTime\": 1535515853.141275, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515852.391501}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #progress_metric: host=algo-1, completed 13 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1184, \"sum\": 1184.0, \"min\": 1184}, \"Total Records Seen\": {\"count\": 1, \"max\": 1178410, \"sum\": 1178410.0, \"min\": 1178410}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 14, \"sum\": 14.0, \"min\": 14}}, \"EndTime\": 1535515853.141481, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 12}, \"StartTime\": 1535515852.391964}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120816.380732 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=13, batch=0 train binary_classification_accuracy <score>=0.727\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=13, batch=0 train binary_classification_cross_entropy <loss>=0.596375854492\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=13, batch=0 train binary_f_1.000 <score>=0.752493200363\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=13, train binary_classification_accuracy <score>=0.719813186813\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=13, train binary_classification_cross_entropy <loss>=0.596616698548\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=13, train binary_f_1.000 <score>=0.767250586506\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 764.2920017242432, \"sum\": 764.2920017242432, \"min\": 764.2920017242432}}, \"EndTime\": 1535515853.906119, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515853.14134}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #progress_metric: host=algo-1, completed 14 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1275, \"sum\": 1275.0, \"min\": 1275}, \"Total Records Seen\": {\"count\": 1, \"max\": 1268980, \"sum\": 1268980.0, \"min\": 1268980}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 15, \"sum\": 15.0, \"min\": 15}}, \"EndTime\": 1535515853.906337, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 13}, \"StartTime\": 1535515853.14179}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=118443.229337 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=14, batch=0 train binary_classification_accuracy <score>=0.725\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=14, batch=0 train binary_classification_cross_entropy <loss>=0.592293945313\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=14, batch=0 train binary_f_1.000 <score>=0.750227066303\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=14, train binary_classification_accuracy <score>=0.726307692308\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=14, train binary_classification_cross_entropy <loss>=0.592896536691\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=14, train binary_f_1.000 <score>=0.770692544239\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 746.225118637085, \"sum\": 746.225118637085, \"min\": 746.225118637085}}, \"EndTime\": 1535515854.652905, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515853.906191}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #progress_metric: host=algo-1, completed 15 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1366, \"sum\": 1366.0, \"min\": 1366}, \"Total Records Seen\": {\"count\": 1, \"max\": 1359550, \"sum\": 1359550.0, \"min\": 1359550}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 16, \"sum\": 16.0, \"min\": 16}}, \"EndTime\": 1535515854.653129, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 14}, \"StartTime\": 1535515853.906646}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121307.821973 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=15, batch=0 train binary_classification_accuracy <score>=0.728\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=15, batch=0 train binary_classification_cross_entropy <loss>=0.588498291016\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=15, batch=0 train binary_f_1.000 <score>=0.753176043557\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=15, train binary_classification_accuracy <score>=0.727472527473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=15, train binary_classification_cross_entropy <loss>=0.58944051621\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=15, train binary_f_1.000 <score>=0.770705819264\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 708.035945892334, \"sum\": 708.035945892334, \"min\": 708.035945892334}}, \"EndTime\": 1535515855.361456, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515854.652987}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #progress_metric: host=algo-1, completed 16 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1457, \"sum\": 1457.0, \"min\": 1457}, \"Total Records Seen\": {\"count\": 1, \"max\": 1450120, \"sum\": 1450120.0, \"min\": 1450120}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 17, \"sum\": 17.0, \"min\": 17}}, \"EndTime\": 1535515855.361672, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 15}, \"StartTime\": 1535515854.653391}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127854.420413 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=16, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=16, batch=0 train binary_classification_cross_entropy <loss>=0.584962280273\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=16, batch=0 train binary_f_1.000 <score>=0.756119673617\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=16, train binary_classification_accuracy <score>=0.729186813187\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=16, train binary_classification_cross_entropy <loss>=0.586223113008\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=16, train binary_f_1.000 <score>=0.771319340052\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 739.9110794067383, \"sum\": 739.9110794067383, \"min\": 739.9110794067383}}, \"EndTime\": 1535515856.101883, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515855.361532}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #progress_metric: host=algo-1, completed 17 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1548, \"sum\": 1548.0, \"min\": 1548}, \"Total Records Seen\": {\"count\": 1, \"max\": 1540690, \"sum\": 1540690.0, \"min\": 1540690}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 18, \"sum\": 18.0, \"min\": 18}}, \"EndTime\": 1535515856.102105, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 16}, \"StartTime\": 1535515855.361944}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122346.440824 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=17, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=17, batch=0 train binary_classification_cross_entropy <loss>=0.581661987305\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=17, batch=0 train binary_f_1.000 <score>=0.756119673617\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=17, train binary_classification_accuracy <score>=0.730549450549\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=17, train binary_classification_cross_entropy <loss>=0.583221362439\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=17, train binary_f_1.000 <score>=0.771698851046\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 752.8820037841797, \"sum\": 752.8820037841797, \"min\": 752.8820037841797}}, \"EndTime\": 1535515856.855296, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515856.101949}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #progress_metric: host=algo-1, completed 18 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1639, \"sum\": 1639.0, \"min\": 1639}, \"Total Records Seen\": {\"count\": 1, \"max\": 1631260, \"sum\": 1631260.0, \"min\": 1631260}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 19, \"sum\": 19.0, \"min\": 19}}, \"EndTime\": 1535515856.855493, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 17}, \"StartTime\": 1535515856.102387}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120244.882344 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=18, batch=0 train binary_classification_accuracy <score>=0.73\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=18, batch=0 train binary_classification_cross_entropy <loss>=0.578575683594\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=18, batch=0 train binary_f_1.000 <score>=0.755434782609\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=18, train binary_classification_accuracy <score>=0.731351648352\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=18, train binary_classification_cross_entropy <loss>=0.580414623219\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=18, train binary_f_1.000 <score>=0.771704720549\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 761.9071006774902, \"sum\": 761.9071006774902, \"min\": 761.9071006774902}}, \"EndTime\": 1535515857.619053, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515856.855361}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #progress_metric: host=algo-1, completed 19 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1730, \"sum\": 1730.0, \"min\": 1730}, \"Total Records Seen\": {\"count\": 1, \"max\": 1721830, \"sum\": 1721830.0, \"min\": 1721830}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 20, \"sum\": 20.0, \"min\": 20}}, \"EndTime\": 1535515857.619258, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 18}, \"StartTime\": 1535515856.85712}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=118819.441995 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=19, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=19, batch=0 train binary_classification_cross_entropy <loss>=0.575684020996\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=19, batch=0 train binary_f_1.000 <score>=0.756561085973\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #quality_metric: host=algo-1, epoch=19, train binary_classification_accuracy <score>=0.732483516484\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #quality_metric: host=algo-1, epoch=19, train binary_classification_cross_entropy <loss>=0.577784366817\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #quality_metric: host=algo-1, epoch=19, train binary_f_1.000 <score>=0.77193179689\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 784.7499847412109, \"sum\": 784.7499847412109, \"min\": 784.7499847412109}}, \"EndTime\": 1535515858.404311, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515857.619117}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #progress_metric: host=algo-1, completed 20 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1821, \"sum\": 1821.0, \"min\": 1821}, \"Total Records Seen\": {\"count\": 1, \"max\": 1812400, \"sum\": 1812400.0, \"min\": 1812400}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 21, \"sum\": 21.0, \"min\": 21}}, \"EndTime\": 1535515858.404594, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 19}, \"StartTime\": 1535515857.619531}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=115349.368178 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #quality_metric: host=algo-1, epoch=20, batch=0 train binary_classification_accuracy <score>=0.732\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #quality_metric: host=algo-1, epoch=20, batch=0 train binary_classification_cross_entropy <loss>=0.572969360352\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:58 INFO 139937213605696] #quality_metric: host=algo-1, epoch=20, batch=0 train binary_f_1.000 <score>=0.757246376812\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=20, train binary_classification_accuracy <score>=0.733054945055\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=20, train binary_classification_cross_entropy <loss>=0.575314023867\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=20, train binary_f_1.000 <score>=0.77187183051\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 746.3319301605225, \"sum\": 746.3319301605225, \"min\": 746.3319301605225}}, \"EndTime\": 1535515859.151346, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515858.404445}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #progress_metric: host=algo-1, completed 21 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 1912, \"sum\": 1912.0, \"min\": 1912}, \"Total Records Seen\": {\"count\": 1, \"max\": 1902970, \"sum\": 1902970.0, \"min\": 1902970}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 22, \"sum\": 22.0, \"min\": 22}}, \"EndTime\": 1535515859.151562, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 20}, \"StartTime\": 1535515858.404981}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121295.117352 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=21, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=21, batch=0 train binary_classification_cross_entropy <loss>=0.57041595459\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=21, batch=0 train binary_f_1.000 <score>=0.756119673617\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=21, train binary_classification_accuracy <score>=0.733659340659\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=21, train binary_classification_cross_entropy <loss>=0.572988758129\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=21, train binary_f_1.000 <score>=0.771906379695\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 757.8160762786865, \"sum\": 757.8160762786865, \"min\": 757.8160762786865}}, \"EndTime\": 1535515859.90967, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515859.151411}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #progress_metric: host=algo-1, completed 22 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2003, \"sum\": 2003.0, \"min\": 2003}, \"Total Records Seen\": {\"count\": 1, \"max\": 1993540, \"sum\": 1993540.0, \"min\": 1993540}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 23, \"sum\": 23.0, \"min\": 23}}, \"EndTime\": 1535515859.909875, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 21}, \"StartTime\": 1535515859.151825}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119459.465335 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=22, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=22, batch=0 train binary_classification_cross_entropy <loss>=0.568009643555\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:10:59 INFO 139937213605696] #quality_metric: host=algo-1, epoch=22, batch=0 train binary_f_1.000 <score>=0.756119673617\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #quality_metric: host=algo-1, epoch=22, train binary_classification_accuracy <score>=0.734373626374\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #quality_metric: host=algo-1, epoch=22, train binary_classification_cross_entropy <loss>=0.570795304854\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #quality_metric: host=algo-1, epoch=22, train binary_f_1.000 <score>=0.772099864233\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 722.2340106964111, \"sum\": 722.2340106964111, \"min\": 722.2340106964111}}, \"EndTime\": 1535515860.632455, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515859.909732}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #progress_metric: host=algo-1, completed 23 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2094, \"sum\": 2094.0, \"min\": 2094}, \"Total Records Seen\": {\"count\": 1, \"max\": 2084110, \"sum\": 2084110.0, \"min\": 2084110}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 24, \"sum\": 24.0, \"min\": 24}}, \"EndTime\": 1535515860.632683, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 22}, \"StartTime\": 1535515859.910188}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=125335.748013 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #quality_metric: host=algo-1, epoch=23, batch=0 train binary_classification_accuracy <score>=0.731\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #quality_metric: host=algo-1, epoch=23, batch=0 train binary_classification_cross_entropy <loss>=0.565737731934\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:00 INFO 139937213605696] #quality_metric: host=algo-1, epoch=23, batch=0 train binary_f_1.000 <score>=0.756119673617\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #quality_metric: host=algo-1, epoch=23, train binary_classification_accuracy <score>=0.735351648352\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #quality_metric: host=algo-1, epoch=23, train binary_classification_cross_entropy <loss>=0.568721807794\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #quality_metric: host=algo-1, epoch=23, train binary_f_1.000 <score>=0.772486372611\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 726.8369197845459, \"sum\": 726.8369197845459, \"min\": 726.8369197845459}}, \"EndTime\": 1535515861.35985, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515860.632526}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #progress_metric: host=algo-1, completed 24 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2185, \"sum\": 2185.0, \"min\": 2185}, \"Total Records Seen\": {\"count\": 1, \"max\": 2174680, \"sum\": 2174680.0, \"min\": 2174680}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 25, \"sum\": 25.0, \"min\": 25}}, \"EndTime\": 1535515861.360071, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 23}, \"StartTime\": 1535515860.632977}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124543.626805 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #quality_metric: host=algo-1, epoch=24, batch=0 train binary_classification_accuracy <score>=0.733\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #quality_metric: host=algo-1, epoch=24, batch=0 train binary_classification_cross_entropy <loss>=0.563588684082\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:01 INFO 139937213605696] #quality_metric: host=algo-1, epoch=24, batch=0 train binary_f_1.000 <score>=0.757932910245\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=24, train binary_classification_accuracy <score>=0.736032967033\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=24, train binary_classification_cross_entropy <loss>=0.56675768104\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=24, train binary_f_1.000 <score>=0.772719961396\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 715.2881622314453, \"sum\": 715.2881622314453, \"min\": 715.2881622314453}}, \"EndTime\": 1535515862.075694, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515861.35992}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #progress_metric: host=algo-1, completed 25 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2276, \"sum\": 2276.0, \"min\": 2276}, \"Total Records Seen\": {\"count\": 1, \"max\": 2265250, \"sum\": 2265250.0, \"min\": 2265250}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 26, \"sum\": 26.0, \"min\": 26}}, \"EndTime\": 1535515862.075902, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 24}, \"StartTime\": 1535515861.360371}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=126556.642534 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=25, batch=0 train binary_classification_accuracy <score>=0.733\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=25, batch=0 train binary_classification_cross_entropy <loss>=0.561552246094\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=25, batch=0 train binary_f_1.000 <score>=0.757932910245\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=25, train binary_classification_accuracy <score>=0.736967032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=25, train binary_classification_cross_entropy <loss>=0.564893442678\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=25, train binary_f_1.000 <score>=0.773148586917\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 696.0811614990234, \"sum\": 696.0811614990234, \"min\": 696.0811614990234}}, \"EndTime\": 1535515862.77237, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515862.075766}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #progress_metric: host=algo-1, completed 26 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2367, \"sum\": 2367.0, \"min\": 2367}, \"Total Records Seen\": {\"count\": 1, \"max\": 2355820, \"sum\": 2355820.0, \"min\": 2355820}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 27, \"sum\": 27.0, \"min\": 27}}, \"EndTime\": 1535515862.772587, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 25}, \"StartTime\": 1535515862.076255}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=130046.521365 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=26, batch=0 train binary_classification_accuracy <score>=0.734\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=26, batch=0 train binary_classification_cross_entropy <loss>=0.559619262695\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:02 INFO 139937213605696] #quality_metric: host=algo-1, epoch=26, batch=0 train binary_f_1.000 <score>=0.758620689655\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #quality_metric: host=algo-1, epoch=26, train binary_classification_accuracy <score>=0.73732967033\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #quality_metric: host=algo-1, epoch=26, train binary_classification_cross_entropy <loss>=0.563120650407\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #quality_metric: host=algo-1, epoch=26, train binary_f_1.000 <score>=0.773110839005\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 763.5347843170166, \"sum\": 763.5347843170166, \"min\": 763.5347843170166}}, \"EndTime\": 1535515863.536433, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515862.772453}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #progress_metric: host=algo-1, completed 27 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2458, \"sum\": 2458.0, \"min\": 2458}, \"Total Records Seen\": {\"count\": 1, \"max\": 2446390, \"sum\": 2446390.0, \"min\": 2446390}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 28, \"sum\": 28.0, \"min\": 28}}, \"EndTime\": 1535515863.536639, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 26}, \"StartTime\": 1535515862.772856}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=118566.333174 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #quality_metric: host=algo-1, epoch=27, batch=0 train binary_classification_accuracy <score>=0.735\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #quality_metric: host=algo-1, epoch=27, batch=0 train binary_classification_cross_entropy <loss>=0.55778137207\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:03 INFO 139937213605696] #quality_metric: host=algo-1, epoch=27, batch=0 train binary_f_1.000 <score>=0.759309718438\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #quality_metric: host=algo-1, epoch=27, train binary_classification_accuracy <score>=0.737516483516\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #quality_metric: host=algo-1, epoch=27, train binary_classification_cross_entropy <loss>=0.561431744586\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #quality_metric: host=algo-1, epoch=27, train binary_f_1.000 <score>=0.77298992587\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 727.8430461883545, \"sum\": 727.8430461883545, \"min\": 727.8430461883545}}, \"EndTime\": 1535515864.264861, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515863.536489}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #progress_metric: host=algo-1, completed 28 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2549, \"sum\": 2549.0, \"min\": 2549}, \"Total Records Seen\": {\"count\": 1, \"max\": 2536960, \"sum\": 2536960.0, \"min\": 2536960}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 29, \"sum\": 29.0, \"min\": 29}}, \"EndTime\": 1535515864.265081, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 27}, \"StartTime\": 1535515863.536994}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124375.830239 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #quality_metric: host=algo-1, epoch=28, batch=0 train binary_classification_accuracy <score>=0.735\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #quality_metric: host=algo-1, epoch=28, batch=0 train binary_classification_cross_entropy <loss>=0.556031005859\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:04 INFO 139937213605696] #quality_metric: host=algo-1, epoch=28, batch=0 train binary_f_1.000 <score>=0.759309718438\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=28, train binary_classification_accuracy <score>=0.738\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=28, train binary_classification_cross_entropy <loss>=0.559819969764\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=28, train binary_f_1.000 <score>=0.77311484146\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 763.2291316986084, \"sum\": 763.2291316986084, \"min\": 763.2291316986084}}, \"EndTime\": 1535515865.028635, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515864.264928}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #progress_metric: host=algo-1, completed 29 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2640, \"sum\": 2640.0, \"min\": 2640}, \"Total Records Seen\": {\"count\": 1, \"max\": 2627530, \"sum\": 2627530.0, \"min\": 2627530}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 30, \"sum\": 30.0, \"min\": 30}}, \"EndTime\": 1535515865.028834, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 28}, \"StartTime\": 1535515864.265375}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=118612.164983 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=29, batch=0 train binary_classification_accuracy <score>=0.736\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=29, batch=0 train binary_classification_cross_entropy <loss>=0.554361694336\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=29, batch=0 train binary_f_1.000 <score>=0.760435571688\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=29, train binary_classification_accuracy <score>=0.738197802198\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=29, train binary_classification_cross_entropy <loss>=0.558279289497\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=29, train binary_f_1.000 <score>=0.773104761905\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 740.3061389923096, \"sum\": 740.3061389923096, \"min\": 740.3061389923096}}, \"EndTime\": 1535515865.769481, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515865.028701}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #progress_metric: host=algo-1, completed 30 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2731, \"sum\": 2731.0, \"min\": 2731}, \"Total Records Seen\": {\"count\": 1, \"max\": 2718100, \"sum\": 2718100.0, \"min\": 2718100}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 31, \"sum\": 31.0, \"min\": 31}}, \"EndTime\": 1535515865.76971, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 29}, \"StartTime\": 1535515865.029142}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122277.877206 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=30, batch=0 train binary_classification_accuracy <score>=0.736\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=30, batch=0 train binary_classification_cross_entropy <loss>=0.552767211914\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:05 INFO 139937213605696] #quality_metric: host=algo-1, epoch=30, batch=0 train binary_f_1.000 <score>=0.760435571688\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #quality_metric: host=algo-1, epoch=30, train binary_classification_accuracy <score>=0.73856043956\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #quality_metric: host=algo-1, epoch=30, train binary_classification_cross_entropy <loss>=0.55680429446\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #quality_metric: host=algo-1, epoch=30, train binary_f_1.000 <score>=0.773148986889\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 717.0147895812988, \"sum\": 717.0147895812988, \"min\": 717.0147895812988}}, \"EndTime\": 1535515866.487052, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515865.769551}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #progress_metric: host=algo-1, completed 31 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2822, \"sum\": 2822.0, \"min\": 2822}, \"Total Records Seen\": {\"count\": 1, \"max\": 2808670, \"sum\": 2808670.0, \"min\": 2808670}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 32, \"sum\": 32.0, \"min\": 32}}, \"EndTime\": 1535515866.487253, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 30}, \"StartTime\": 1535515865.770009}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=126255.64578 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #quality_metric: host=algo-1, epoch=31, batch=0 train binary_classification_accuracy <score>=0.736\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #quality_metric: host=algo-1, epoch=31, batch=0 train binary_classification_cross_entropy <loss>=0.5512421875\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:06 INFO 139937213605696] #quality_metric: host=algo-1, epoch=31, batch=0 train binary_f_1.000 <score>=0.760869565217\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=31, train binary_classification_accuracy <score>=0.73889010989\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=31, train binary_classification_cross_entropy <loss>=0.555390148121\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=31, train binary_f_1.000 <score>=0.77329236993\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 730.7028770446777, \"sum\": 730.7028770446777, \"min\": 730.7028770446777}}, \"EndTime\": 1535515867.218276, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515866.487118}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #progress_metric: host=algo-1, completed 32 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 2913, \"sum\": 2913.0, \"min\": 2913}, \"Total Records Seen\": {\"count\": 1, \"max\": 2899240, \"sum\": 2899240.0, \"min\": 2899240}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 33, \"sum\": 33.0, \"min\": 33}}, \"EndTime\": 1535515867.218505, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 31}, \"StartTime\": 1535515866.487538}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123883.019933 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=32, batch=0 train binary_classification_accuracy <score>=0.738\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=32, batch=0 train binary_classification_cross_entropy <loss>=0.549781799316\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=32, batch=0 train binary_f_1.000 <score>=0.76268115942\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=32, train binary_classification_accuracy <score>=0.739142857143\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=32, train binary_classification_cross_entropy <loss>=0.554032518995\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=32, train binary_f_1.000 <score>=0.773289018776\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 753.3588409423828, \"sum\": 753.3588409423828, \"min\": 753.3588409423828}}, \"EndTime\": 1535515867.972167, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515867.218349}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #progress_metric: host=algo-1, completed 33 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3004, \"sum\": 3004.0, \"min\": 3004}, \"Total Records Seen\": {\"count\": 1, \"max\": 2989810, \"sum\": 2989810.0, \"min\": 2989810}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 34, \"sum\": 34.0, \"min\": 34}}, \"EndTime\": 1535515867.972377, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 32}, \"StartTime\": 1535515867.218775}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120164.283523 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=33, batch=0 train binary_classification_accuracy <score>=0.739\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=33, batch=0 train binary_classification_cross_entropy <loss>=0.548381530762\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:07 INFO 139937213605696] #quality_metric: host=algo-1, epoch=33, batch=0 train binary_f_1.000 <score>=0.763372620127\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #quality_metric: host=algo-1, epoch=33, train binary_classification_accuracy <score>=0.739472527473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #quality_metric: host=algo-1, epoch=33, train binary_classification_cross_entropy <loss>=0.552727505527\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #quality_metric: host=algo-1, epoch=33, train binary_f_1.000 <score>=0.773367746869\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 757.3380470275879, \"sum\": 757.3380470275879, \"min\": 757.3380470275879}}, \"EndTime\": 1535515868.730033, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515867.972239}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #progress_metric: host=algo-1, completed 34 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3095, \"sum\": 3095.0, \"min\": 3095}, \"Total Records Seen\": {\"count\": 1, \"max\": 3080380, \"sum\": 3080380.0, \"min\": 3080380}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 35, \"sum\": 35.0, \"min\": 35}}, \"EndTime\": 1535515868.730253, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 33}, \"StartTime\": 1535515867.972668}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119530.432389 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #quality_metric: host=algo-1, epoch=34, batch=0 train binary_classification_accuracy <score>=0.739\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #quality_metric: host=algo-1, epoch=34, batch=0 train binary_classification_cross_entropy <loss>=0.547037475586\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:08 INFO 139937213605696] #quality_metric: host=algo-1, epoch=34, batch=0 train binary_f_1.000 <score>=0.763372620127\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #quality_metric: host=algo-1, epoch=34, train binary_classification_accuracy <score>=0.739901098901\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #quality_metric: host=algo-1, epoch=34, train binary_classification_cross_entropy <loss>=0.551471594507\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #quality_metric: host=algo-1, epoch=34, train binary_f_1.000 <score>=0.773500224883\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 688.6539459228516, \"sum\": 688.6539459228516, \"min\": 688.6539459228516}}, \"EndTime\": 1535515869.419276, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515868.73009}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #progress_metric: host=algo-1, completed 35 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3186, \"sum\": 3186.0, \"min\": 3186}, \"Total Records Seen\": {\"count\": 1, \"max\": 3170950, \"sum\": 3170950.0, \"min\": 3170950}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 36, \"sum\": 36.0, \"min\": 36}}, \"EndTime\": 1535515869.419483, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 34}, \"StartTime\": 1535515868.730592}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=131447.351344 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #quality_metric: host=algo-1, epoch=35, batch=0 train binary_classification_accuracy <score>=0.74\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #quality_metric: host=algo-1, epoch=35, batch=0 train binary_classification_cross_entropy <loss>=0.545745910645\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:09 INFO 139937213605696] #quality_metric: host=algo-1, epoch=35, batch=0 train binary_f_1.000 <score>=0.764492753623\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=35, train binary_classification_accuracy <score>=0.740021978022\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=35, train binary_classification_cross_entropy <loss>=0.550261639606\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=35, train binary_f_1.000 <score>=0.773447225787\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 737.7021312713623, \"sum\": 737.7021312713623, \"min\": 737.7021312713623}}, \"EndTime\": 1535515870.157512, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515869.419337}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #progress_metric: host=algo-1, completed 36 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3277, \"sum\": 3277.0, \"min\": 3277}, \"Total Records Seen\": {\"count\": 1, \"max\": 3261520, \"sum\": 3261520.0, \"min\": 3261520}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 37, \"sum\": 37.0, \"min\": 37}}, \"EndTime\": 1535515870.157707, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 35}, \"StartTime\": 1535515869.419779}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122715.74156 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=36, batch=0 train binary_classification_accuracy <score>=0.742\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=36, batch=0 train binary_classification_cross_entropy <loss>=0.544503662109\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=36, batch=0 train binary_f_1.000 <score>=0.766304347826\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=36, train binary_classification_accuracy <score>=0.740417582418\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=36, train binary_classification_cross_entropy <loss>=0.549094782903\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=36, train binary_f_1.000 <score>=0.773653245434\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 739.2170429229736, \"sum\": 739.2170429229736, \"min\": 739.2170429229736}}, \"EndTime\": 1535515870.897248, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515870.157571}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #progress_metric: host=algo-1, completed 37 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3368, \"sum\": 3368.0, \"min\": 3368}, \"Total Records Seen\": {\"count\": 1, \"max\": 3352090, \"sum\": 3352090.0, \"min\": 3352090}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 38, \"sum\": 38.0, \"min\": 38}}, \"EndTime\": 1535515870.897471, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 36}, \"StartTime\": 1535515870.157994}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122460.581887 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=37, batch=0 train binary_classification_accuracy <score>=0.743\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=37, batch=0 train binary_classification_cross_entropy <loss>=0.543307861328\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:10 INFO 139937213605696] #quality_metric: host=algo-1, epoch=37, batch=0 train binary_f_1.000 <score>=0.76742081448\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #quality_metric: host=algo-1, epoch=37, train binary_classification_accuracy <score>=0.740857142857\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #quality_metric: host=algo-1, epoch=37, train binary_classification_cross_entropy <loss>=0.547968435434\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #quality_metric: host=algo-1, epoch=37, train binary_f_1.000 <score>=0.773897869566\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 759.3941688537598, \"sum\": 759.3941688537598, \"min\": 759.3941688537598}}, \"EndTime\": 1535515871.657331, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515870.897319}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #progress_metric: host=algo-1, completed 38 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3459, \"sum\": 3459.0, \"min\": 3459}, \"Total Records Seen\": {\"count\": 1, \"max\": 3442660, \"sum\": 3442660.0, \"min\": 3442660}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 39, \"sum\": 39.0, \"min\": 39}}, \"EndTime\": 1535515871.657553, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 37}, \"StartTime\": 1535515870.897908}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119209.01198 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #quality_metric: host=algo-1, epoch=38, batch=0 train binary_classification_accuracy <score>=0.742\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #quality_metric: host=algo-1, epoch=38, batch=0 train binary_classification_cross_entropy <loss>=0.542155517578\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:11 INFO 139937213605696] #quality_metric: host=algo-1, epoch=38, batch=0 train binary_f_1.000 <score>=0.766304347826\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #quality_metric: host=algo-1, epoch=38, train binary_classification_accuracy <score>=0.741175824176\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #quality_metric: host=algo-1, epoch=38, train binary_classification_cross_entropy <loss>=0.546880263779\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #quality_metric: host=algo-1, epoch=38, train binary_f_1.000 <score>=0.774039430134\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 736.5999221801758, \"sum\": 736.5999221801758, \"min\": 736.5999221801758}}, \"EndTime\": 1535515872.395828, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515871.657408}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #progress_metric: host=algo-1, completed 39 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3550, \"sum\": 3550.0, \"min\": 3550}, \"Total Records Seen\": {\"count\": 1, \"max\": 3533230, \"sum\": 3533230.0, \"min\": 3533230}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 40, \"sum\": 40.0, \"min\": 40}}, \"EndTime\": 1535515872.396038, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 38}, \"StartTime\": 1535515871.659196}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122895.623053 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #quality_metric: host=algo-1, epoch=39, batch=0 train binary_classification_accuracy <score>=0.743\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #quality_metric: host=algo-1, epoch=39, batch=0 train binary_classification_cross_entropy <loss>=0.541044433594\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:12 INFO 139937213605696] #quality_metric: host=algo-1, epoch=39, batch=0 train binary_f_1.000 <score>=0.76742081448\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=39, train binary_classification_accuracy <score>=0.741450549451\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=39, train binary_classification_cross_entropy <loss>=0.545828132378\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=39, train binary_f_1.000 <score>=0.774103731014\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 747.0309734344482, \"sum\": 747.0309734344482, \"min\": 747.0309734344482}}, \"EndTime\": 1535515873.143391, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515872.395904}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #progress_metric: host=algo-1, completed 40 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3641, \"sum\": 3641.0, \"min\": 3641}, \"Total Records Seen\": {\"count\": 1, \"max\": 3623800, \"sum\": 3623800.0, \"min\": 3623800}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 41, \"sum\": 41.0, \"min\": 41}}, \"EndTime\": 1535515873.143578, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 39}, \"StartTime\": 1535515872.396329}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121188.820534 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=40, batch=0 train binary_classification_accuracy <score>=0.743\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=40, batch=0 train binary_classification_cross_entropy <loss>=0.539972167969\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=40, batch=0 train binary_f_1.000 <score>=0.76742081448\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=40, train binary_classification_accuracy <score>=0.741714285714\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=40, train binary_classification_cross_entropy <loss>=0.544810086764\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=40, train binary_f_1.000 <score>=0.774217098943\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 714.8690223693848, \"sum\": 714.8690223693848, \"min\": 714.8690223693848}}, \"EndTime\": 1535515873.858738, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515873.143447}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #progress_metric: host=algo-1, completed 41 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3732, \"sum\": 3732.0, \"min\": 3732}, \"Total Records Seen\": {\"count\": 1, \"max\": 3714370, \"sum\": 3714370.0, \"min\": 3714370}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 42, \"sum\": 42.0, \"min\": 42}}, \"EndTime\": 1535515873.85898, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 40}, \"StartTime\": 1535515873.143838}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=126628.485911 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=41, batch=0 train binary_classification_accuracy <score>=0.742\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=41, batch=0 train binary_classification_cross_entropy <loss>=0.538936706543\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:13 INFO 139937213605696] #quality_metric: host=algo-1, epoch=41, batch=0 train binary_f_1.000 <score>=0.766726943942\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #quality_metric: host=algo-1, epoch=41, train binary_classification_accuracy <score>=0.741879120879\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #quality_metric: host=algo-1, epoch=41, train binary_classification_cross_entropy <loss>=0.543824348869\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #quality_metric: host=algo-1, epoch=41, train binary_f_1.000 <score>=0.774263610591\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 732.5799465179443, \"sum\": 732.5799465179443, \"min\": 732.5799465179443}}, \"EndTime\": 1535515874.591873, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515873.858809}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #progress_metric: host=algo-1, completed 42 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3823, \"sum\": 3823.0, \"min\": 3823}, \"Total Records Seen\": {\"count\": 1, \"max\": 3804940, \"sum\": 3804940.0, \"min\": 3804940}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 43, \"sum\": 43.0, \"min\": 43}}, \"EndTime\": 1535515874.592086, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 41}, \"StartTime\": 1535515873.859262}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123569.826433 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #quality_metric: host=algo-1, epoch=42, batch=0 train binary_classification_accuracy <score>=0.744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #quality_metric: host=algo-1, epoch=42, batch=0 train binary_classification_cross_entropy <loss>=0.537936157227\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:14 INFO 139937213605696] #quality_metric: host=algo-1, epoch=42, batch=0 train binary_f_1.000 <score>=0.768953068592\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #quality_metric: host=algo-1, epoch=42, train binary_classification_accuracy <score>=0.74210989011\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #quality_metric: host=algo-1, epoch=42, train binary_classification_cross_entropy <loss>=0.542869268061\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #quality_metric: host=algo-1, epoch=42, train binary_f_1.000 <score>=0.774363510499\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 737.0791435241699, \"sum\": 737.0791435241699, \"min\": 737.0791435241699}}, \"EndTime\": 1535515875.330818, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515874.591944}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #progress_metric: host=algo-1, completed 43 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 3914, \"sum\": 3914.0, \"min\": 3914}, \"Total Records Seen\": {\"count\": 1, \"max\": 3895510, \"sum\": 3895510.0, \"min\": 3895510}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 44, \"sum\": 44.0, \"min\": 44}}, \"EndTime\": 1535515875.331081, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 42}, \"StartTime\": 1535515874.593706}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122809.44762 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #quality_metric: host=algo-1, epoch=43, batch=0 train binary_classification_accuracy <score>=0.744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #quality_metric: host=algo-1, epoch=43, batch=0 train binary_classification_cross_entropy <loss>=0.53696875\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:15 INFO 139937213605696] #quality_metric: host=algo-1, epoch=43, batch=0 train binary_f_1.000 <score>=0.768953068592\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=43, train binary_classification_accuracy <score>=0.742054945055\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=43, train binary_classification_cross_entropy <loss>=0.541943342607\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=43, train binary_f_1.000 <score>=0.774213406951\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 723.046064376831, \"sum\": 723.046064376831, \"min\": 723.046064376831}}, \"EndTime\": 1535515876.054537, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515875.3309}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #progress_metric: host=algo-1, completed 44 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4005, \"sum\": 4005.0, \"min\": 4005}, \"Total Records Seen\": {\"count\": 1, \"max\": 3986080, \"sum\": 3986080.0, \"min\": 3986080}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 45, \"sum\": 45.0, \"min\": 45}}, \"EndTime\": 1535515876.054747, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 43}, \"StartTime\": 1535515875.33146}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=125203.723461 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=44, batch=0 train binary_classification_accuracy <score>=0.744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=44, batch=0 train binary_classification_cross_entropy <loss>=0.536032714844\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=44, batch=0 train binary_f_1.000 <score>=0.768953068592\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=44, train binary_classification_accuracy <score>=0.74221978022\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=44, train binary_classification_cross_entropy <loss>=0.54104516065\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=44, train binary_f_1.000 <score>=0.774281700441\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 752.1700859069824, \"sum\": 752.1700859069824, \"min\": 752.1700859069824}}, \"EndTime\": 1535515876.80723, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515876.054597}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #progress_metric: host=algo-1, completed 45 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4096, \"sum\": 4096.0, \"min\": 4096}, \"Total Records Seen\": {\"count\": 1, \"max\": 4076650, \"sum\": 4076650.0, \"min\": 4076650}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 46, \"sum\": 46.0, \"min\": 46}}, \"EndTime\": 1535515876.807432, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 44}, \"StartTime\": 1535515876.055035}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120357.078833 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=45, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=45, batch=0 train binary_classification_cross_entropy <loss>=0.53512677002\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:16 INFO 139937213605696] #quality_metric: host=algo-1, epoch=45, batch=0 train binary_f_1.000 <score>=0.77034358047\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #quality_metric: host=algo-1, epoch=45, train binary_classification_accuracy <score>=0.742472527473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #quality_metric: host=algo-1, epoch=45, train binary_classification_cross_entropy <loss>=0.540173438439\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #quality_metric: host=algo-1, epoch=45, train binary_f_1.000 <score>=0.774400985762\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 741.063117980957, \"sum\": 741.063117980957, \"min\": 741.063117980957}}, \"EndTime\": 1535515877.548841, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515876.807289}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #progress_metric: host=algo-1, completed 46 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4187, \"sum\": 4187.0, \"min\": 4187}, \"Total Records Seen\": {\"count\": 1, \"max\": 4167220, \"sum\": 4167220.0, \"min\": 4167220}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 47, \"sum\": 47.0, \"min\": 47}}, \"EndTime\": 1535515877.549035, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 45}, \"StartTime\": 1535515876.807746}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122158.812158 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #quality_metric: host=algo-1, epoch=46, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #quality_metric: host=algo-1, epoch=46, batch=0 train binary_classification_cross_entropy <loss>=0.534249328613\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:17 INFO 139937213605696] #quality_metric: host=algo-1, epoch=46, batch=0 train binary_f_1.000 <score>=0.77034358047\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #quality_metric: host=algo-1, epoch=46, train binary_classification_accuracy <score>=0.742626373626\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #quality_metric: host=algo-1, epoch=46, train binary_classification_cross_entropy <loss>=0.539326961978\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #quality_metric: host=algo-1, epoch=46, train binary_f_1.000 <score>=0.774431528156\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 769.3729400634766, \"sum\": 769.3729400634766, \"min\": 769.3729400634766}}, \"EndTime\": 1535515878.318772, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515877.548906}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #progress_metric: host=algo-1, completed 47 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4278, \"sum\": 4278.0, \"min\": 4278}, \"Total Records Seen\": {\"count\": 1, \"max\": 4257790, \"sum\": 4257790.0, \"min\": 4257790}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 48, \"sum\": 48.0, \"min\": 48}}, \"EndTime\": 1535515878.319029, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 46}, \"StartTime\": 1535515877.549366}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=117650.883262 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #quality_metric: host=algo-1, epoch=47, batch=0 train binary_classification_accuracy <score>=0.747\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #quality_metric: host=algo-1, epoch=47, batch=0 train binary_classification_cross_entropy <loss>=0.533399047852\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:18 INFO 139937213605696] #quality_metric: host=algo-1, epoch=47, batch=0 train binary_f_1.000 <score>=0.771866546438\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=47, train binary_classification_accuracy <score>=0.742692307692\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=47, train binary_classification_cross_entropy <loss>=0.53850458971\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=47, train binary_f_1.000 <score>=0.774415445533\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 729.1219234466553, \"sum\": 729.1219234466553, \"min\": 729.1219234466553}}, \"EndTime\": 1535515879.048467, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515878.318838}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #progress_metric: host=algo-1, completed 48 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4369, \"sum\": 4369.0, \"min\": 4369}, \"Total Records Seen\": {\"count\": 1, \"max\": 4348360, \"sum\": 4348360.0, \"min\": 4348360}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 49, \"sum\": 49.0, \"min\": 49}}, \"EndTime\": 1535515879.048688, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 47}, \"StartTime\": 1535515878.319311}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124155.914457 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=48, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=48, batch=0 train binary_classification_cross_entropy <loss>=0.532574768066\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=48, batch=0 train binary_f_1.000 <score>=0.770758122744\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=48, train binary_classification_accuracy <score>=0.742824175824\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=48, train binary_classification_cross_entropy <loss>=0.537705274645\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=48, train binary_f_1.000 <score>=0.774465870653\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 746.4599609375, \"sum\": 746.4599609375, \"min\": 746.4599609375}}, \"EndTime\": 1535515879.79548, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515879.048524}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #progress_metric: host=algo-1, completed 49 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4460, \"sum\": 4460.0, \"min\": 4460}, \"Total Records Seen\": {\"count\": 1, \"max\": 4438930, \"sum\": 4438930.0, \"min\": 4438930}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 50, \"sum\": 50.0, \"min\": 50}}, \"EndTime\": 1535515879.795677, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 48}, \"StartTime\": 1535515879.048991}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121277.149507 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=49, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=49, batch=0 train binary_classification_cross_entropy <loss>=0.531775268555\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:19 INFO 139937213605696] #quality_metric: host=algo-1, epoch=49, batch=0 train binary_f_1.000 <score>=0.770758122744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #quality_metric: host=algo-1, epoch=49, train binary_classification_accuracy <score>=0.742846153846\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #quality_metric: host=algo-1, epoch=49, train binary_classification_cross_entropy <loss>=0.536928018088\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #quality_metric: host=algo-1, epoch=49, train binary_f_1.000 <score>=0.774415578156\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 749.6969699859619, \"sum\": 749.6969699859619, \"min\": 749.6969699859619}}, \"EndTime\": 1535515880.547029, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515879.795545}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #progress_metric: host=algo-1, completed 50 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4551, \"sum\": 4551.0, \"min\": 4551}, \"Total Records Seen\": {\"count\": 1, \"max\": 4529500, \"sum\": 4529500.0, \"min\": 4529500}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 51, \"sum\": 51.0, \"min\": 51}}, \"EndTime\": 1535515880.547241, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 49}, \"StartTime\": 1535515879.797304}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120750.825829 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #quality_metric: host=algo-1, epoch=50, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #quality_metric: host=algo-1, epoch=50, batch=0 train binary_classification_cross_entropy <loss>=0.530999511719\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:20 INFO 139937213605696] #quality_metric: host=algo-1, epoch=50, batch=0 train binary_f_1.000 <score>=0.770758122744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #quality_metric: host=algo-1, epoch=50, train binary_classification_accuracy <score>=0.742956043956\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #quality_metric: host=algo-1, epoch=50, train binary_classification_cross_entropy <loss>=0.536171892774\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #quality_metric: host=algo-1, epoch=50, train binary_f_1.000 <score>=0.77442934704\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 756.3290596008301, \"sum\": 756.3290596008301, \"min\": 756.3290596008301}}, \"EndTime\": 1535515881.304118, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515880.547098}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #progress_metric: host=algo-1, completed 51 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4642, \"sum\": 4642.0, \"min\": 4642}, \"Total Records Seen\": {\"count\": 1, \"max\": 4620070, \"sum\": 4620070.0, \"min\": 4620070}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 52, \"sum\": 52.0, \"min\": 52}}, \"EndTime\": 1535515881.304322, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 50}, \"StartTime\": 1535515880.547755}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119689.81358 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #quality_metric: host=algo-1, epoch=51, batch=0 train binary_classification_accuracy <score>=0.745\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #quality_metric: host=algo-1, epoch=51, batch=0 train binary_classification_cross_entropy <loss>=0.530246337891\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:21 INFO 139937213605696] #quality_metric: host=algo-1, epoch=51, batch=0 train binary_f_1.000 <score>=0.769647696477\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=51, train binary_classification_accuracy <score>=0.742923076923\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=51, train binary_classification_cross_entropy <loss>=0.535436004639\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=51, train binary_f_1.000 <score>=0.77433295391\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 741.5330410003662, \"sum\": 741.5330410003662, \"min\": 741.5330410003662}}, \"EndTime\": 1535515882.04623, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515881.304182}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #progress_metric: host=algo-1, completed 52 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4733, \"sum\": 4733.0, \"min\": 4733}, \"Total Records Seen\": {\"count\": 1, \"max\": 4710640, \"sum\": 4710640.0, \"min\": 4710640}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 53, \"sum\": 53.0, \"min\": 53}}, \"EndTime\": 1535515882.046461, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 51}, \"StartTime\": 1535515881.304668}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122075.000363 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=52, batch=0 train binary_classification_accuracy <score>=0.745\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=52, batch=0 train binary_classification_cross_entropy <loss>=0.529514892578\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=52, batch=0 train binary_f_1.000 <score>=0.769647696477\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=52, train binary_classification_accuracy <score>=0.743153846154\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=52, train binary_classification_cross_entropy <loss>=0.534719514281\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=52, train binary_f_1.000 <score>=0.774441967517\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 758.9280605316162, \"sum\": 758.9280605316162, \"min\": 758.9280605316162}}, \"EndTime\": 1535515882.805735, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515882.046304}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #progress_metric: host=algo-1, completed 53 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4824, \"sum\": 4824.0, \"min\": 4824}, \"Total Records Seen\": {\"count\": 1, \"max\": 4801210, \"sum\": 4801210.0, \"min\": 4801210}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 54, \"sum\": 54.0, \"min\": 54}}, \"EndTime\": 1535515882.805951, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 52}, \"StartTime\": 1535515882.046774}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119280.318392 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=53, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=53, batch=0 train binary_classification_cross_entropy <loss>=0.528804077148\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:22 INFO 139937213605696] #quality_metric: host=algo-1, epoch=53, batch=0 train binary_f_1.000 <score>=0.770758122744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #quality_metric: host=algo-1, epoch=53, train binary_classification_accuracy <score>=0.74332967033\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #quality_metric: host=algo-1, epoch=53, train binary_classification_cross_entropy <loss>=0.534021624219\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #quality_metric: host=algo-1, epoch=53, train binary_f_1.000 <score>=0.774561564373\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 792.0980453491211, \"sum\": 792.0980453491211, \"min\": 792.0980453491211}}, \"EndTime\": 1535515883.598405, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515882.805796}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #progress_metric: host=algo-1, completed 54 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 4915, \"sum\": 4915.0, \"min\": 4915}, \"Total Records Seen\": {\"count\": 1, \"max\": 4891780, \"sum\": 4891780.0, \"min\": 4891780}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 55, \"sum\": 55.0, \"min\": 55}}, \"EndTime\": 1535515883.598608, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 53}, \"StartTime\": 1535515882.806265}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=114290.407411 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #quality_metric: host=algo-1, epoch=54, batch=0 train binary_classification_accuracy <score>=0.746\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #quality_metric: host=algo-1, epoch=54, batch=0 train binary_classification_cross_entropy <loss>=0.528113220215\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:23 INFO 139937213605696] #quality_metric: host=algo-1, epoch=54, batch=0 train binary_f_1.000 <score>=0.770758122744\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #quality_metric: host=algo-1, epoch=54, train binary_classification_accuracy <score>=0.743450549451\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #quality_metric: host=algo-1, epoch=54, train binary_classification_cross_entropy <loss>=0.533341581911\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #quality_metric: host=algo-1, epoch=54, train binary_f_1.000 <score>=0.774600293504\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 738.6949062347412, \"sum\": 738.6949062347412, \"min\": 738.6949062347412}}, \"EndTime\": 1535515884.337633, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515883.598465}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #progress_metric: host=algo-1, completed 55 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5006, \"sum\": 5006.0, \"min\": 5006}, \"Total Records Seen\": {\"count\": 1, \"max\": 4982350, \"sum\": 4982350.0, \"min\": 4982350}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 56, \"sum\": 56.0, \"min\": 56}}, \"EndTime\": 1535515884.337885, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 54}, \"StartTime\": 1535515883.598909}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122543.105716 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #quality_metric: host=algo-1, epoch=55, batch=0 train binary_classification_accuracy <score>=0.75\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #quality_metric: host=algo-1, epoch=55, batch=0 train binary_classification_cross_entropy <loss>=0.52744128418\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:24 INFO 139937213605696] #quality_metric: host=algo-1, epoch=55, batch=0 train binary_f_1.000 <score>=0.774774774775\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=55, train binary_classification_accuracy <score>=0.743802197802\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=55, train binary_classification_cross_entropy <loss>=0.532678658622\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=55, train binary_f_1.000 <score>=0.774917937826\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 751.4209747314453, \"sum\": 751.4209747314453, \"min\": 751.4209747314453}}, \"EndTime\": 1535515885.089594, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515884.337707}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #progress_metric: host=algo-1, completed 56 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5097, \"sum\": 5097.0, \"min\": 5097}, \"Total Records Seen\": {\"count\": 1, \"max\": 5072920, \"sum\": 5072920.0, \"min\": 5072920}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 57, \"sum\": 57.0, \"min\": 57}}, \"EndTime\": 1535515885.089792, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 55}, \"StartTime\": 1535515884.338142}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120475.521439 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=56, batch=0 train binary_classification_accuracy <score>=0.75\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=56, batch=0 train binary_classification_cross_entropy <loss>=0.526787475586\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=56, batch=0 train binary_f_1.000 <score>=0.774774774775\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=56, train binary_classification_accuracy <score>=0.744021978022\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=56, train binary_classification_cross_entropy <loss>=0.532032178271\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=56, train binary_f_1.000 <score>=0.775071937583\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 737.0200157165527, \"sum\": 737.0200157165527, \"min\": 737.0200157165527}}, \"EndTime\": 1535515885.827165, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515885.08966}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #progress_metric: host=algo-1, completed 57 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5188, \"sum\": 5188.0, \"min\": 5188}, \"Total Records Seen\": {\"count\": 1, \"max\": 5163490, \"sum\": 5163490.0, \"min\": 5163490}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 58, \"sum\": 58.0, \"min\": 58}}, \"EndTime\": 1535515885.827393, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 56}, \"StartTime\": 1535515885.090111}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122822.630308 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=57, batch=0 train binary_classification_accuracy <score>=0.751\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=57, batch=0 train binary_classification_cross_entropy <loss>=0.526151245117\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:25 INFO 139937213605696] #quality_metric: host=algo-1, epoch=57, batch=0 train binary_f_1.000 <score>=0.775877587759\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #quality_metric: host=algo-1, epoch=57, train binary_classification_accuracy <score>=0.744087912088\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #quality_metric: host=algo-1, epoch=57, train binary_classification_cross_entropy <loss>=0.531401479868\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #quality_metric: host=algo-1, epoch=57, train binary_f_1.000 <score>=0.775103814582\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 770.3251838684082, \"sum\": 770.3251838684082, \"min\": 770.3251838684082}}, \"EndTime\": 1535515886.59811, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515885.827242}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #progress_metric: host=algo-1, completed 58 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5279, \"sum\": 5279.0, \"min\": 5279}, \"Total Records Seen\": {\"count\": 1, \"max\": 5254060, \"sum\": 5254060.0, \"min\": 5254060}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 59, \"sum\": 59.0, \"min\": 59}}, \"EndTime\": 1535515886.598304, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 57}, \"StartTime\": 1535515885.827753}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=117520.364084 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #quality_metric: host=algo-1, epoch=58, batch=0 train binary_classification_accuracy <score>=0.751\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #quality_metric: host=algo-1, epoch=58, batch=0 train binary_classification_cross_entropy <loss>=0.525531616211\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:26 INFO 139937213605696] #quality_metric: host=algo-1, epoch=58, batch=0 train binary_f_1.000 <score>=0.775877587759\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #quality_metric: host=algo-1, epoch=58, train binary_classification_accuracy <score>=0.744087912088\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #quality_metric: host=algo-1, epoch=58, train binary_classification_cross_entropy <loss>=0.530785925897\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #quality_metric: host=algo-1, epoch=58, train binary_f_1.000 <score>=0.775060368975\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 767.2300338745117, \"sum\": 767.2300338745117, \"min\": 767.2300338745117}}, \"EndTime\": 1535515887.365892, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515886.598174}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #progress_metric: host=algo-1, completed 59 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5370, \"sum\": 5370.0, \"min\": 5370}, \"Total Records Seen\": {\"count\": 1, \"max\": 5344630, \"sum\": 5344630.0, \"min\": 5344630}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 60, \"sum\": 60.0, \"min\": 60}}, \"EndTime\": 1535515887.366067, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 58}, \"StartTime\": 1535515886.598629}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=117997.435925 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #quality_metric: host=algo-1, epoch=59, batch=0 train binary_classification_accuracy <score>=0.751\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #quality_metric: host=algo-1, epoch=59, batch=0 train binary_classification_cross_entropy <loss>=0.524927978516\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:27 INFO 139937213605696] #quality_metric: host=algo-1, epoch=59, batch=0 train binary_f_1.000 <score>=0.775877587759\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=59, train binary_classification_accuracy <score>=0.744274725275\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=59, train binary_classification_cross_entropy <loss>=0.530184921768\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=59, train binary_f_1.000 <score>=0.775178969945\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 768.1949138641357, \"sum\": 768.1949138641357, \"min\": 768.1949138641357}}, \"EndTime\": 1535515888.135926, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515887.36595}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #progress_metric: host=algo-1, completed 60 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5461, \"sum\": 5461.0, \"min\": 5461}, \"Total Records Seen\": {\"count\": 1, \"max\": 5435200, \"sum\": 5435200.0, \"min\": 5435200}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 61, \"sum\": 61.0, \"min\": 61}}, \"EndTime\": 1535515888.136133, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 59}, \"StartTime\": 1535515887.367701}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=117846.365671 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=60, batch=0 train binary_classification_accuracy <score>=0.749\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=60, batch=0 train binary_classification_cross_entropy <loss>=0.524339599609\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=60, batch=0 train binary_f_1.000 <score>=0.774077407741\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=60, train binary_classification_accuracy <score>=0.744351648352\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=60, train binary_classification_cross_entropy <loss>=0.529597887647\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=60, train binary_f_1.000 <score>=0.77522705314\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 776.6289710998535, \"sum\": 776.6289710998535, \"min\": 776.6289710998535}}, \"EndTime\": 1535515888.913077, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515888.135992}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #progress_metric: host=algo-1, completed 61 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5552, \"sum\": 5552.0, \"min\": 5552}, \"Total Records Seen\": {\"count\": 1, \"max\": 5525770, \"sum\": 5525770.0, \"min\": 5525770}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 62, \"sum\": 62.0, \"min\": 62}}, \"EndTime\": 1535515888.913293, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 60}, \"StartTime\": 1535515888.136414}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=116563.249034 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=61, batch=0 train binary_classification_accuracy <score>=0.749\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=61, batch=0 train binary_classification_cross_entropy <loss>=0.523765930176\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:28 INFO 139937213605696] #quality_metric: host=algo-1, epoch=61, batch=0 train binary_f_1.000 <score>=0.774077407741\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #quality_metric: host=algo-1, epoch=61, train binary_classification_accuracy <score>=0.744538461538\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #quality_metric: host=algo-1, epoch=61, train binary_classification_cross_entropy <loss>=0.529024258456\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #quality_metric: host=algo-1, epoch=61, train binary_f_1.000 <score>=0.775358747645\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 731.8141460418701, \"sum\": 731.8141460418701, \"min\": 731.8141460418701}}, \"EndTime\": 1535515889.646829, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515888.913151}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #progress_metric: host=algo-1, completed 62 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5643, \"sum\": 5643.0, \"min\": 5643}, \"Total Records Seen\": {\"count\": 1, \"max\": 5616340, \"sum\": 5616340.0, \"min\": 5616340}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 63, \"sum\": 63.0, \"min\": 63}}, \"EndTime\": 1535515889.647022, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 61}, \"StartTime\": 1535515888.914986}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123700.601013 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #quality_metric: host=algo-1, epoch=62, batch=0 train binary_classification_accuracy <score>=0.75\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #quality_metric: host=algo-1, epoch=62, batch=0 train binary_classification_cross_entropy <loss>=0.523206298828\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:29 INFO 139937213605696] #quality_metric: host=algo-1, epoch=62, batch=0 train binary_f_1.000 <score>=0.775179856115\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #quality_metric: host=algo-1, epoch=62, train binary_classification_accuracy <score>=0.744692307692\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #quality_metric: host=algo-1, epoch=62, train binary_classification_cross_entropy <loss>=0.528463500306\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #quality_metric: host=algo-1, epoch=62, train binary_f_1.000 <score>=0.775467996482\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 756.7639350891113, \"sum\": 756.7639350891113, \"min\": 756.7639350891113}}, \"EndTime\": 1535515890.404083, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515889.646876}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #progress_metric: host=algo-1, completed 63 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5734, \"sum\": 5734.0, \"min\": 5734}, \"Total Records Seen\": {\"count\": 1, \"max\": 5706910, \"sum\": 5706910.0, \"min\": 5706910}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 64, \"sum\": 64.0, \"min\": 64}}, \"EndTime\": 1535515890.404311, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 62}, \"StartTime\": 1535515889.647285}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=119618.657202 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #quality_metric: host=algo-1, epoch=63, batch=0 train binary_classification_accuracy <score>=0.75\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #quality_metric: host=algo-1, epoch=63, batch=0 train binary_classification_cross_entropy <loss>=0.52266003418\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:30 INFO 139937213605696] #quality_metric: host=algo-1, epoch=63, batch=0 train binary_f_1.000 <score>=0.775179856115\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=63, train binary_classification_accuracy <score>=0.744725274725\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=63, train binary_classification_cross_entropy <loss>=0.527915116866\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=63, train binary_f_1.000 <score>=0.775468780205\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 714.1110897064209, \"sum\": 714.1110897064209, \"min\": 714.1110897064209}}, \"EndTime\": 1535515891.11873, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515890.404147}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #progress_metric: host=algo-1, completed 64 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5825, \"sum\": 5825.0, \"min\": 5825}, \"Total Records Seen\": {\"count\": 1, \"max\": 5797480, \"sum\": 5797480.0, \"min\": 5797480}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 65, \"sum\": 65.0, \"min\": 65}}, \"EndTime\": 1535515891.11893, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 63}, \"StartTime\": 1535515890.404589}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=126765.479788 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=64, batch=0 train binary_classification_accuracy <score>=0.75\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=64, batch=0 train binary_classification_cross_entropy <loss>=0.522126708984\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=64, batch=0 train binary_f_1.000 <score>=0.775179856115\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=64, train binary_classification_accuracy <score>=0.744868131868\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=64, train binary_classification_cross_entropy <loss>=0.527378596379\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=64, train binary_f_1.000 <score>=0.775548874216\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 720.181941986084, \"sum\": 720.181941986084, \"min\": 720.181941986084}}, \"EndTime\": 1535515891.839463, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515891.118795}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #progress_metric: host=algo-1, completed 65 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 5916, \"sum\": 5916.0, \"min\": 5916}, \"Total Records Seen\": {\"count\": 1, \"max\": 5888050, \"sum\": 5888050.0, \"min\": 5888050}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 66, \"sum\": 66.0, \"min\": 66}}, \"EndTime\": 1535515891.839681, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 64}, \"StartTime\": 1535515891.119248}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=125697.219109 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=65, batch=0 train binary_classification_accuracy <score>=0.751\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=65, batch=0 train binary_classification_cross_entropy <loss>=0.521605651855\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:31 INFO 139937213605696] #quality_metric: host=algo-1, epoch=65, batch=0 train binary_f_1.000 <score>=0.77628032345\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #quality_metric: host=algo-1, epoch=65, train binary_classification_accuracy <score>=0.744868131868\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #quality_metric: host=algo-1, epoch=65, train binary_classification_cross_entropy <loss>=0.526853469681\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #quality_metric: host=algo-1, epoch=65, train binary_f_1.000 <score>=0.775544534354\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 746.7679977416992, \"sum\": 746.7679977416992, \"min\": 746.7679977416992}}, \"EndTime\": 1535515892.586917, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515891.839546}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #progress_metric: host=algo-1, completed 66 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6007, \"sum\": 6007.0, \"min\": 6007}, \"Total Records Seen\": {\"count\": 1, \"max\": 5978620, \"sum\": 5978620.0, \"min\": 5978620}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 67, \"sum\": 67.0, \"min\": 67}}, \"EndTime\": 1535515892.587171, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 65}, \"StartTime\": 1535515891.840115}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121216.237874 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #quality_metric: host=algo-1, epoch=66, batch=0 train binary_classification_accuracy <score>=0.751\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #quality_metric: host=algo-1, epoch=66, batch=0 train binary_classification_cross_entropy <loss>=0.521096435547\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:32 INFO 139937213605696] #quality_metric: host=algo-1, epoch=66, batch=0 train binary_f_1.000 <score>=0.77628032345\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #quality_metric: host=algo-1, epoch=66, train binary_classification_accuracy <score>=0.745098901099\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #quality_metric: host=algo-1, epoch=66, train binary_classification_cross_entropy <loss>=0.526339271965\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #quality_metric: host=algo-1, epoch=66, train binary_f_1.000 <score>=0.77570201903\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 735.569953918457, \"sum\": 735.569953918457, \"min\": 735.569953918457}}, \"EndTime\": 1535515893.323164, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515892.587008}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #progress_metric: host=algo-1, completed 67 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6098, \"sum\": 6098.0, \"min\": 6098}, \"Total Records Seen\": {\"count\": 1, \"max\": 6069190, \"sum\": 6069190.0, \"min\": 6069190}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 68, \"sum\": 68.0, \"min\": 68}}, \"EndTime\": 1535515893.323379, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 66}, \"StartTime\": 1535515892.587569}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123073.281347 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #quality_metric: host=algo-1, epoch=67, batch=0 train binary_classification_accuracy <score>=0.752\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #quality_metric: host=algo-1, epoch=67, batch=0 train binary_classification_cross_entropy <loss>=0.520598388672\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:33 INFO 139937213605696] #quality_metric: host=algo-1, epoch=67, batch=0 train binary_f_1.000 <score>=0.777378815081\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=67, train binary_classification_accuracy <score>=0.745296703297\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=67, train binary_classification_cross_entropy <loss>=0.525835572966\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=67, train binary_f_1.000 <score>=0.77584139265\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 753.8459300994873, \"sum\": 753.8459300994873, \"min\": 753.8459300994873}}, \"EndTime\": 1535515894.077572, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515893.323227}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #progress_metric: host=algo-1, completed 68 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6189, \"sum\": 6189.0, \"min\": 6189}, \"Total Records Seen\": {\"count\": 1, \"max\": 6159760, \"sum\": 6159760.0, \"min\": 6159760}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 69, \"sum\": 69.0, \"min\": 69}}, \"EndTime\": 1535515894.077769, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 67}, \"StartTime\": 1535515893.323693}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120086.070313 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=68, batch=0 train binary_classification_accuracy <score>=0.752\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=68, batch=0 train binary_classification_cross_entropy <loss>=0.52011126709\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=68, batch=0 train binary_f_1.000 <score>=0.777378815081\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=68, train binary_classification_accuracy <score>=0.745505494505\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=68, train binary_classification_cross_entropy <loss>=0.525341927329\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=68, train binary_f_1.000 <score>=0.775996982212\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 751.1529922485352, \"sum\": 751.1529922485352, \"min\": 751.1529922485352}}, \"EndTime\": 1535515894.829409, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515894.077636}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #progress_metric: host=algo-1, completed 69 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6280, \"sum\": 6280.0, \"min\": 6280}, \"Total Records Seen\": {\"count\": 1, \"max\": 6250330, \"sum\": 6250330.0, \"min\": 6250330}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 70, \"sum\": 70.0, \"min\": 70}}, \"EndTime\": 1535515894.82962, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 68}, \"StartTime\": 1535515894.078224}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120515.576649 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=69, batch=0 train binary_classification_accuracy <score>=0.752\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=69, batch=0 train binary_classification_cross_entropy <loss>=0.519634399414\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:34 INFO 139937213605696] #quality_metric: host=algo-1, epoch=69, batch=0 train binary_f_1.000 <score>=0.777378815081\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #quality_metric: host=algo-1, epoch=69, train binary_classification_accuracy <score>=0.745725274725\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #quality_metric: host=algo-1, epoch=69, train binary_classification_cross_entropy <loss>=0.524857952747\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #quality_metric: host=algo-1, epoch=69, train binary_f_1.000 <score>=0.776181770697\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 697.1080303192139, \"sum\": 697.1080303192139, \"min\": 697.1080303192139}}, \"EndTime\": 1535515895.527084, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515894.829466}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #progress_metric: host=algo-1, completed 70 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6371, \"sum\": 6371.0, \"min\": 6371}, \"Total Records Seen\": {\"count\": 1, \"max\": 6340900, \"sum\": 6340900.0, \"min\": 6340900}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 71, \"sum\": 71.0, \"min\": 71}}, \"EndTime\": 1535515895.527294, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 69}, \"StartTime\": 1535515894.829945}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=129857.098807 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #quality_metric: host=algo-1, epoch=70, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #quality_metric: host=algo-1, epoch=70, batch=0 train binary_classification_cross_entropy <loss>=0.519167419434\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:35 INFO 139937213605696] #quality_metric: host=algo-1, epoch=70, batch=0 train binary_f_1.000 <score>=0.778077268643\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #quality_metric: host=algo-1, epoch=70, train binary_classification_accuracy <score>=0.745967032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #quality_metric: host=algo-1, epoch=70, train binary_classification_cross_entropy <loss>=0.524383234716\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #quality_metric: host=algo-1, epoch=70, train binary_f_1.000 <score>=0.77639024579\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 742.3291206359863, \"sum\": 742.3291206359863, \"min\": 742.3291206359863}}, \"EndTime\": 1535515896.269955, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515895.52715}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #progress_metric: host=algo-1, completed 71 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6462, \"sum\": 6462.0, \"min\": 6462}, \"Total Records Seen\": {\"count\": 1, \"max\": 6431470, \"sum\": 6431470.0, \"min\": 6431470}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 72, \"sum\": 72.0, \"min\": 72}}, \"EndTime\": 1535515896.270168, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 70}, \"StartTime\": 1535515895.527594}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121945.642072 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #quality_metric: host=algo-1, epoch=71, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #quality_metric: host=algo-1, epoch=71, batch=0 train binary_classification_cross_entropy <loss>=0.518709838867\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:36 INFO 139937213605696] #quality_metric: host=algo-1, epoch=71, batch=0 train binary_f_1.000 <score>=0.778077268643\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=71, train binary_classification_accuracy <score>=0.746164835165\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=71, train binary_classification_cross_entropy <loss>=0.523917387575\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=71, train binary_f_1.000 <score>=0.776547067416\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 753.7610530853271, \"sum\": 753.7610530853271, \"min\": 753.7610530853271}}, \"EndTime\": 1535515897.02429, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515896.270023}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #progress_metric: host=algo-1, completed 72 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6553, \"sum\": 6553.0, \"min\": 6553}, \"Total Records Seen\": {\"count\": 1, \"max\": 6522040, \"sum\": 6522040.0, \"min\": 6522040}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 73, \"sum\": 73.0, \"min\": 73}}, \"EndTime\": 1535515897.024497, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 71}, \"StartTime\": 1535515896.270497}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120100.117635 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=72, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=72, batch=0 train binary_classification_cross_entropy <loss>=0.518261352539\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=72, batch=0 train binary_f_1.000 <score>=0.778077268643\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=72, train binary_classification_accuracy <score>=0.746351648352\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=72, train binary_classification_cross_entropy <loss>=0.523460050813\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=72, train binary_f_1.000 <score>=0.776709360368\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 739.1958236694336, \"sum\": 739.1958236694336, \"min\": 739.1958236694336}}, \"EndTime\": 1535515897.764025, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515897.024355}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #progress_metric: host=algo-1, completed 73 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6644, \"sum\": 6644.0, \"min\": 6644}, \"Total Records Seen\": {\"count\": 1, \"max\": 6612610, \"sum\": 6612610.0, \"min\": 6612610}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 74, \"sum\": 74.0, \"min\": 74}}, \"EndTime\": 1535515897.764313, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 72}, \"StartTime\": 1535515897.024795}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122450.160745 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=73, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=73, batch=0 train binary_classification_cross_entropy <loss>=0.517821411133\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:37 INFO 139937213605696] #quality_metric: host=algo-1, epoch=73, batch=0 train binary_f_1.000 <score>=0.778077268643\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #quality_metric: host=algo-1, epoch=73, train binary_classification_accuracy <score>=0.746417582418\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #quality_metric: host=algo-1, epoch=73, train binary_classification_cross_entropy <loss>=0.523010872977\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #quality_metric: host=algo-1, epoch=73, train binary_f_1.000 <score>=0.776737165967\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 726.9899845123291, \"sum\": 726.9899845123291, \"min\": 726.9899845123291}}, \"EndTime\": 1535515898.491635, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515897.76409}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #progress_metric: host=algo-1, completed 74 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6735, \"sum\": 6735.0, \"min\": 6735}, \"Total Records Seen\": {\"count\": 1, \"max\": 6703180, \"sum\": 6703180.0, \"min\": 6703180}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 75, \"sum\": 75.0, \"min\": 75}}, \"EndTime\": 1535515898.491833, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 73}, \"StartTime\": 1535515897.764612}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124520.520283 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #quality_metric: host=algo-1, epoch=74, batch=0 train binary_classification_accuracy <score>=0.752\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #quality_metric: host=algo-1, epoch=74, batch=0 train binary_classification_cross_entropy <loss>=0.517389709473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:38 INFO 139937213605696] #quality_metric: host=algo-1, epoch=74, batch=0 train binary_f_1.000 <score>=0.777378815081\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=74, train binary_classification_accuracy <score>=0.746714285714\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=74, train binary_classification_cross_entropy <loss>=0.522569498586\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=74, train binary_f_1.000 <score>=0.776978974156\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 751.24192237854, \"sum\": 751.24192237854, \"min\": 751.24192237854}}, \"EndTime\": 1535515899.243372, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515898.491701}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #progress_metric: host=algo-1, completed 75 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6826, \"sum\": 6826.0, \"min\": 6826}, \"Total Records Seen\": {\"count\": 1, \"max\": 6793750, \"sum\": 6793750.0, \"min\": 6793750}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 76, \"sum\": 76.0, \"min\": 76}}, \"EndTime\": 1535515899.243602, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 74}, \"StartTime\": 1535515898.492103}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120502.158245 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=75, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=75, batch=0 train binary_classification_cross_entropy <loss>=0.516965942383\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=75, batch=0 train binary_f_1.000 <score>=0.778077268643\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=75, train binary_classification_accuracy <score>=0.746835164835\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=75, train binary_classification_cross_entropy <loss>=0.522135612404\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=75, train binary_f_1.000 <score>=0.777035789637\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 709.3179225921631, \"sum\": 709.3179225921631, \"min\": 709.3179225921631}}, \"EndTime\": 1535515899.953237, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515899.24344}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #progress_metric: host=algo-1, completed 76 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 6917, \"sum\": 6917.0, \"min\": 6917}, \"Total Records Seen\": {\"count\": 1, \"max\": 6884320, \"sum\": 6884320.0, \"min\": 6884320}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 77, \"sum\": 77.0, \"min\": 77}}, \"EndTime\": 1535515899.953445, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 75}, \"StartTime\": 1535515899.24389}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127623.243242 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=76, batch=0 train binary_classification_accuracy <score>=0.754\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=76, batch=0 train binary_classification_cross_entropy <loss>=0.516549560547\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:39 INFO 139937213605696] #quality_metric: host=algo-1, epoch=76, batch=0 train binary_f_1.000 <score>=0.779174147217\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #quality_metric: host=algo-1, epoch=76, train binary_classification_accuracy <score>=0.746989010989\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #quality_metric: host=algo-1, epoch=76, train binary_classification_cross_entropy <loss>=0.521708878067\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #quality_metric: host=algo-1, epoch=76, train binary_f_1.000 <score>=0.777171283123\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 738.0011081695557, \"sum\": 738.0011081695557, \"min\": 738.0011081695557}}, \"EndTime\": 1535515900.691752, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515899.953293}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #progress_metric: host=algo-1, completed 77 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7008, \"sum\": 7008.0, \"min\": 7008}, \"Total Records Seen\": {\"count\": 1, \"max\": 6974890, \"sum\": 6974890.0, \"min\": 6974890}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 78, \"sum\": 78.0, \"min\": 78}}, \"EndTime\": 1535515900.692089, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 76}, \"StartTime\": 1535515899.953722}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122638.686157 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #quality_metric: host=algo-1, epoch=77, batch=0 train binary_classification_accuracy <score>=0.754\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #quality_metric: host=algo-1, epoch=77, batch=0 train binary_classification_cross_entropy <loss>=0.516140380859\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:40 INFO 139937213605696] #quality_metric: host=algo-1, epoch=77, batch=0 train binary_f_1.000 <score>=0.779174147217\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #quality_metric: host=algo-1, epoch=77, train binary_classification_accuracy <score>=0.747241758242\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #quality_metric: host=algo-1, epoch=77, train binary_classification_cross_entropy <loss>=0.521288997441\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #quality_metric: host=algo-1, epoch=77, train binary_f_1.000 <score>=0.77737017858\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 732.8588962554932, \"sum\": 732.8588962554932, \"min\": 732.8588962554932}}, \"EndTime\": 1535515901.425421, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515900.691813}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #progress_metric: host=algo-1, completed 78 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7099, \"sum\": 7099.0, \"min\": 7099}, \"Total Records Seen\": {\"count\": 1, \"max\": 7065460, \"sum\": 7065460.0, \"min\": 7065460}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 79, \"sum\": 79.0, \"min\": 79}}, \"EndTime\": 1535515901.425644, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 77}, \"StartTime\": 1535515900.69253}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123517.393181 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #quality_metric: host=algo-1, epoch=78, batch=0 train binary_classification_accuracy <score>=0.755\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #quality_metric: host=algo-1, epoch=78, batch=0 train binary_classification_cross_entropy <loss>=0.515738037109\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:41 INFO 139937213605696] #quality_metric: host=algo-1, epoch=78, batch=0 train binary_f_1.000 <score>=0.780269058296\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=78, train binary_classification_accuracy <score>=0.747406593407\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=78, train binary_classification_cross_entropy <loss>=0.520875674405\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=78, train binary_f_1.000 <score>=0.777487367137\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 712.3870849609375, \"sum\": 712.3870849609375, \"min\": 712.3870849609375}}, \"EndTime\": 1535515902.13837, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515901.425491}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #progress_metric: host=algo-1, completed 79 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7190, \"sum\": 7190.0, \"min\": 7190}, \"Total Records Seen\": {\"count\": 1, \"max\": 7156030, \"sum\": 7156030.0, \"min\": 7156030}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 80, \"sum\": 80.0, \"min\": 80}}, \"EndTime\": 1535515902.138576, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 78}, \"StartTime\": 1535515901.425951}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127072.1032 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=79, batch=0 train binary_classification_accuracy <score>=0.755\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=79, batch=0 train binary_classification_cross_entropy <loss>=0.515342163086\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=79, batch=0 train binary_f_1.000 <score>=0.780269058296\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=79, train binary_classification_accuracy <score>=0.747681318681\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=79, train binary_classification_cross_entropy <loss>=0.520468629606\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=79, train binary_f_1.000 <score>=0.777718617191\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 718.6279296875, \"sum\": 718.6279296875, \"min\": 718.6279296875}}, \"EndTime\": 1535515902.858897, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515902.138435}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #progress_metric: host=algo-1, completed 80 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7281, \"sum\": 7281.0, \"min\": 7281}, \"Total Records Seen\": {\"count\": 1, \"max\": 7246600, \"sum\": 7246600.0, \"min\": 7246600}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 81, \"sum\": 81.0, \"min\": 81}}, \"EndTime\": 1535515902.859212, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 79}, \"StartTime\": 1535515902.14024}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=125950.481361 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=80, batch=0 train binary_classification_accuracy <score>=0.755\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=80, batch=0 train binary_classification_cross_entropy <loss>=0.514952514648\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:42 INFO 139937213605696] #quality_metric: host=algo-1, epoch=80, batch=0 train binary_f_1.000 <score>=0.780269058296\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=80, train binary_classification_accuracy <score>=0.747813186813\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=80, train binary_classification_cross_entropy <loss>=0.520067563906\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=80, train binary_f_1.000 <score>=0.777804672599\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 725.3918647766113, \"sum\": 725.3918647766113, \"min\": 725.3918647766113}}, \"EndTime\": 1535515903.58499, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515902.859049}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #progress_metric: host=algo-1, completed 81 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7372, \"sum\": 7372.0, \"min\": 7372}, \"Total Records Seen\": {\"count\": 1, \"max\": 7337170, \"sum\": 7337170.0, \"min\": 7337170}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 82, \"sum\": 82.0, \"min\": 82}}, \"EndTime\": 1535515903.585197, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 80}, \"StartTime\": 1535515902.859569}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124798.816685 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=81, batch=0 train binary_classification_accuracy <score>=0.755\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=81, batch=0 train binary_classification_cross_entropy <loss>=0.514568725586\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:43 INFO 139937213605696] #quality_metric: host=algo-1, epoch=81, batch=0 train binary_f_1.000 <score>=0.780269058296\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=81, train binary_classification_accuracy <score>=0.747967032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=81, train binary_classification_cross_entropy <loss>=0.519672232827\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=81, train binary_f_1.000 <score>=0.77792302106\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 718.4479236602783, \"sum\": 718.4479236602783, \"min\": 718.4479236602783}}, \"EndTime\": 1535515904.30396, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515903.58506}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #progress_metric: host=algo-1, completed 82 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7463, \"sum\": 7463.0, \"min\": 7463}, \"Total Records Seen\": {\"count\": 1, \"max\": 7427740, \"sum\": 7427740.0, \"min\": 7427740}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 83, \"sum\": 83.0, \"min\": 83}}, \"EndTime\": 1535515904.304177, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 81}, \"StartTime\": 1535515903.585486}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=126002.200864 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=82, batch=0 train binary_classification_accuracy <score>=0.755\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=82, batch=0 train binary_classification_cross_entropy <loss>=0.514190673828\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:44 INFO 139937213605696] #quality_metric: host=algo-1, epoch=82, batch=0 train binary_f_1.000 <score>=0.780662488809\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=82, train binary_classification_accuracy <score>=0.747967032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=82, train binary_classification_cross_entropy <loss>=0.519282366071\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=82, train binary_f_1.000 <score>=0.777935922386\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 753.3609867095947, \"sum\": 753.3609867095947, \"min\": 753.3609867095947}}, \"EndTime\": 1535515905.057858, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515904.304015}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #progress_metric: host=algo-1, completed 83 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7554, \"sum\": 7554.0, \"min\": 7554}, \"Total Records Seen\": {\"count\": 1, \"max\": 7518310, \"sum\": 7518310.0, \"min\": 7518310}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 84, \"sum\": 84.0, \"min\": 84}}, \"EndTime\": 1535515905.058067, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 82}, \"StartTime\": 1535515904.304462}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=120162.383015 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=83, batch=0 train binary_classification_accuracy <score>=0.754\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=83, batch=0 train binary_classification_cross_entropy <loss>=0.513817810059\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=83, batch=0 train binary_f_1.000 <score>=0.779569892473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=83, train binary_classification_accuracy <score>=0.748054945055\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=83, train binary_classification_cross_entropy <loss>=0.518897714133\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=83, train binary_f_1.000 <score>=0.778021978022\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 743.7100410461426, \"sum\": 743.7100410461426, \"min\": 743.7100410461426}}, \"EndTime\": 1535515905.802085, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515905.057927}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #progress_metric: host=algo-1, completed 84 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7645, \"sum\": 7645.0, \"min\": 7645}, \"Total Records Seen\": {\"count\": 1, \"max\": 7608880, \"sum\": 7608880.0, \"min\": 7608880}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 85, \"sum\": 85.0, \"min\": 85}}, \"EndTime\": 1535515905.802295, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 83}, \"StartTime\": 1535515905.058344}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=121722.409903 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=84, batch=0 train binary_classification_accuracy <score>=0.754\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=84, batch=0 train binary_classification_cross_entropy <loss>=0.513450134277\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:45 INFO 139937213605696] #quality_metric: host=algo-1, epoch=84, batch=0 train binary_f_1.000 <score>=0.779569892473\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=84, train binary_classification_accuracy <score>=0.748153846154\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=84, train binary_classification_cross_entropy <loss>=0.518518053663\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=84, train binary_f_1.000 <score>=0.778089778845\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 730.7980060577393, \"sum\": 730.7980060577393, \"min\": 730.7980060577393}}, \"EndTime\": 1535515906.533404, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515905.802161}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #progress_metric: host=algo-1, completed 85 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7736, \"sum\": 7736.0, \"min\": 7736}, \"Total Records Seen\": {\"count\": 1, \"max\": 7699450, \"sum\": 7699450.0, \"min\": 7699450}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 86, \"sum\": 86.0, \"min\": 86}}, \"EndTime\": 1535515906.533614, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 84}, \"StartTime\": 1535515905.802577}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123873.365131 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=85, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=85, batch=0 train binary_classification_cross_entropy <loss>=0.513087280273\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:46 INFO 139937213605696] #quality_metric: host=algo-1, epoch=85, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=85, train binary_classification_accuracy <score>=0.748252747253\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=85, train binary_classification_cross_entropy <loss>=0.518143140185\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=85, train binary_f_1.000 <score>=0.778183367383\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 721.796989440918, \"sum\": 721.796989440918, \"min\": 721.796989440918}}, \"EndTime\": 1535515907.255726, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515906.53347}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #progress_metric: host=algo-1, completed 86 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7827, \"sum\": 7827.0, \"min\": 7827}, \"Total Records Seen\": {\"count\": 1, \"max\": 7790020, \"sum\": 7790020.0, \"min\": 7790020}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 87, \"sum\": 87.0, \"min\": 87}}, \"EndTime\": 1535515907.255929, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 85}, \"StartTime\": 1535515906.533898}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=125417.514589 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=86, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=86, batch=0 train binary_classification_cross_entropy <loss>=0.512728942871\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=86, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=86, train binary_classification_accuracy <score>=0.748505494505\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=86, train binary_classification_cross_entropy <loss>=0.517772743644\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=86, train binary_f_1.000 <score>=0.778416792533\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 692.7139759063721, \"sum\": 692.7139759063721, \"min\": 692.7139759063721}}, \"EndTime\": 1535515907.949001, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515907.255794}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #progress_metric: host=algo-1, completed 87 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 7918, \"sum\": 7918.0, \"min\": 7918}, \"Total Records Seen\": {\"count\": 1, \"max\": 7880590, \"sum\": 7880590.0, \"min\": 7880590}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 88, \"sum\": 88.0, \"min\": 88}}, \"EndTime\": 1535515907.94933, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 86}, \"StartTime\": 1535515907.256256}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=130657.564002 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=87, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=87, batch=0 train binary_classification_cross_entropy <loss>=0.512375\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:47 INFO 139937213605696] #quality_metric: host=algo-1, epoch=87, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=87, train binary_classification_accuracy <score>=0.748648351648\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=87, train binary_classification_cross_entropy <loss>=0.517406665509\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=87, train binary_f_1.000 <score>=0.778531937761\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 737.9639148712158, \"sum\": 737.9639148712158, \"min\": 737.9639148712158}}, \"EndTime\": 1535515908.687621, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515907.949061}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #progress_metric: host=algo-1, completed 88 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8009, \"sum\": 8009.0, \"min\": 8009}, \"Total Records Seen\": {\"count\": 1, \"max\": 7971160, \"sum\": 7971160.0, \"min\": 7971160}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 89, \"sum\": 89.0, \"min\": 89}}, \"EndTime\": 1535515908.688013, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 87}, \"StartTime\": 1535515907.949621}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122629.541018 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=88, batch=0 train binary_classification_accuracy <score>=0.754\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=88, batch=0 train binary_classification_cross_entropy <loss>=0.512025268555\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:48 INFO 139937213605696] #quality_metric: host=algo-1, epoch=88, batch=0 train binary_f_1.000 <score>=0.779964221825\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=88, train binary_classification_accuracy <score>=0.748857142857\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=88, train binary_classification_cross_entropy <loss>=0.517044693161\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=88, train binary_f_1.000 <score>=0.77873518705\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 739.7232055664062, \"sum\": 739.7232055664062, \"min\": 739.7232055664062}}, \"EndTime\": 1535515909.428254, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515908.687736}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #progress_metric: host=algo-1, completed 89 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8100, \"sum\": 8100.0, \"min\": 8100}, \"Total Records Seen\": {\"count\": 1, \"max\": 8061730, \"sum\": 8061730.0, \"min\": 8061730}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 90, \"sum\": 90.0, \"min\": 90}}, \"EndTime\": 1535515909.428461, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 88}, \"StartTime\": 1535515908.688498}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122380.140376 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=89, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=89, batch=0 train binary_classification_cross_entropy <loss>=0.511679351807\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:49 INFO 139937213605696] #quality_metric: host=algo-1, epoch=89, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=89, train binary_classification_accuracy <score>=0.748967032967\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=89, train binary_classification_cross_entropy <loss>=0.516686616332\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=89, train binary_f_1.000 <score>=0.778832003718\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 734.490156173706, \"sum\": 734.490156173706, \"min\": 734.490156173706}}, \"EndTime\": 1535515910.163265, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515909.428314}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #progress_metric: host=algo-1, completed 90 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8191, \"sum\": 8191.0, \"min\": 8191}, \"Total Records Seen\": {\"count\": 1, \"max\": 8152300, \"sum\": 8152300.0, \"min\": 8152300}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}}, \"EndTime\": 1535515910.163461, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 89}, \"StartTime\": 1535515909.428745}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=123249.494606 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=90, batch=0 train binary_classification_accuracy <score>=0.752\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=90, batch=0 train binary_classification_cross_entropy <loss>=0.511337219238\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=90, batch=0 train binary_f_1.000 <score>=0.778175313059\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=90, train binary_classification_accuracy <score>=0.749010989011\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=90, train binary_classification_cross_entropy <loss>=0.516332265665\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=90, train binary_f_1.000 <score>=0.778892137311\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 736.3810539245605, \"sum\": 736.3810539245605, \"min\": 736.3810539245605}}, \"EndTime\": 1535515910.900156, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515910.163327}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #progress_metric: host=algo-1, completed 91 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8282, \"sum\": 8282.0, \"min\": 8282}, \"Total Records Seen\": {\"count\": 1, \"max\": 8242870, \"sum\": 8242870.0, \"min\": 8242870}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 92, \"sum\": 92.0, \"min\": 92}}, \"EndTime\": 1535515910.900389, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 90}, \"StartTime\": 1535515910.163744}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=122933.484681 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=91, batch=0 train binary_classification_accuracy <score>=0.752\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=91, batch=0 train binary_classification_cross_entropy <loss>=0.510998535156\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:50 INFO 139937213605696] #quality_metric: host=algo-1, epoch=91, batch=0 train binary_f_1.000 <score>=0.778175313059\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=91, train binary_classification_accuracy <score>=0.749175824176\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=91, train binary_classification_cross_entropy <loss>=0.515981431563\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=91, train binary_f_1.000 <score>=0.779052320798\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 712.5329971313477, \"sum\": 712.5329971313477, \"min\": 712.5329971313477}}, \"EndTime\": 1535515911.613318, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515910.90023}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #progress_metric: host=algo-1, completed 92 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8373, \"sum\": 8373.0, \"min\": 8373}, \"Total Records Seen\": {\"count\": 1, \"max\": 8333440, \"sum\": 8333440.0, \"min\": 8333440}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 93, \"sum\": 93.0, \"min\": 93}}, \"EndTime\": 1535515911.613517, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 91}, \"StartTime\": 1535515910.900752}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127047.879068 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=92, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=92, batch=0 train binary_classification_cross_entropy <loss>=0.510663208008\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:51 INFO 139937213605696] #quality_metric: host=algo-1, epoch=92, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=92, train binary_classification_accuracy <score>=0.749307692308\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=92, train binary_classification_cross_entropy <loss>=0.515633942992\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=92, train binary_f_1.000 <score>=0.779155654943\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 729.18701171875, \"sum\": 729.18701171875, \"min\": 729.18701171875}}, \"EndTime\": 1535515912.343073, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515911.613383}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #progress_metric: host=algo-1, completed 93 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8464, \"sum\": 8464.0, \"min\": 8464}, \"Total Records Seen\": {\"count\": 1, \"max\": 8424010, \"sum\": 8424010.0, \"min\": 8424010}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 94, \"sum\": 94.0, \"min\": 94}}, \"EndTime\": 1535515912.343266, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 92}, \"StartTime\": 1535515911.613859}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124150.639542 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=93, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=93, batch=0 train binary_classification_cross_entropy <loss>=0.51033102417\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:52 INFO 139937213605696] #quality_metric: host=algo-1, epoch=93, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=93, train binary_classification_accuracy <score>=0.749230769231\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=93, train binary_classification_cross_entropy <loss>=0.515289630932\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=93, train binary_f_1.000 <score>=0.779081474597\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 717.365026473999, \"sum\": 717.365026473999, \"min\": 717.365026473999}}, \"EndTime\": 1535515913.060992, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515912.343129}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #progress_metric: host=algo-1, completed 94 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8555, \"sum\": 8555.0, \"min\": 8555}, \"Total Records Seen\": {\"count\": 1, \"max\": 8514580, \"sum\": 8514580.0, \"min\": 8514580}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 95, \"sum\": 95.0, \"min\": 95}}, \"EndTime\": 1535515913.061198, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 93}, \"StartTime\": 1535515912.343596}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=126195.1654 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=94, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=94, batch=0 train binary_classification_cross_entropy <loss>=0.510001831055\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=94, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=94, train binary_classification_accuracy <score>=0.74921978022\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=94, train binary_classification_cross_entropy <loss>=0.514948327704\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=94, train binary_f_1.000 <score>=0.779056821152\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 728.7800312042236, \"sum\": 728.7800312042236, \"min\": 728.7800312042236}}, \"EndTime\": 1535515913.790277, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515913.061047}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #progress_metric: host=algo-1, completed 95 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8646, \"sum\": 8646.0, \"min\": 8646}, \"Total Records Seen\": {\"count\": 1, \"max\": 8605150, \"sum\": 8605150.0, \"min\": 8605150}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 96, \"sum\": 96.0, \"min\": 96}}, \"EndTime\": 1535515913.790476, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 94}, \"StartTime\": 1535515913.061469}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=124218.923461 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=95, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=95, batch=0 train binary_classification_cross_entropy <loss>=0.509675415039\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:53 INFO 139937213605696] #quality_metric: host=algo-1, epoch=95, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=95, train binary_classification_accuracy <score>=0.74932967033\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=95, train binary_classification_cross_entropy <loss>=0.514609861604\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=95, train binary_f_1.000 <score>=0.77917074068\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 666.536808013916, \"sum\": 666.536808013916, \"min\": 666.536808013916}}, \"EndTime\": 1535515914.45733, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515913.790343}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #progress_metric: host=algo-1, completed 96 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8737, \"sum\": 8737.0, \"min\": 8737}, \"Total Records Seen\": {\"count\": 1, \"max\": 8695720, \"sum\": 8695720.0, \"min\": 8695720}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 97, \"sum\": 97.0, \"min\": 97}}, \"EndTime\": 1535515914.457531, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 95}, \"StartTime\": 1535515913.790766}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=135812.339106 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=96, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=96, batch=0 train binary_classification_cross_entropy <loss>=0.509351593018\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:54 INFO 139937213605696] #quality_metric: host=algo-1, epoch=96, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=96, train binary_classification_accuracy <score>=0.749417582418\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=96, train binary_classification_cross_entropy <loss>=0.514274082393\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=96, train binary_f_1.000 <score>=0.779265282416\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 704.042911529541, \"sum\": 704.042911529541, \"min\": 704.042911529541}}, \"EndTime\": 1535515915.163152, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515914.457391}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #progress_metric: host=algo-1, completed 97 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8828, \"sum\": 8828.0, \"min\": 8828}, \"Total Records Seen\": {\"count\": 1, \"max\": 8786290, \"sum\": 8786290.0, \"min\": 8786290}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 98, \"sum\": 98.0, \"min\": 98}}, \"EndTime\": 1535515915.163503, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 96}, \"StartTime\": 1535515914.459081}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=128547.784286 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=97, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=97, batch=0 train binary_classification_cross_entropy <loss>=0.509030303955\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=97, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=97, train binary_classification_accuracy <score>=0.749626373626\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=97, train binary_classification_cross_entropy <loss>=0.513940852909\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=97, train binary_f_1.000 <score>=0.779451338741\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 710.982084274292, \"sum\": 710.982084274292, \"min\": 710.982084274292}}, \"EndTime\": 1535515915.876154, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515915.163209}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #progress_metric: host=algo-1, completed 98 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 8919, \"sum\": 8919.0, \"min\": 8919}, \"Total Records Seen\": {\"count\": 1, \"max\": 8876860, \"sum\": 8876860.0, \"min\": 8876860}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 99, \"sum\": 99.0, \"min\": 99}}, \"EndTime\": 1535515915.876354, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 97}, \"StartTime\": 1535515915.165138}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127327.015422 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=98, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=98, batch=0 train binary_classification_cross_entropy <loss>=0.508711334229\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:55 INFO 139937213605696] #quality_metric: host=algo-1, epoch=98, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=98, train binary_classification_accuracy <score>=0.74978021978\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=98, train binary_classification_cross_entropy <loss>=0.513610013522\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=98, train binary_f_1.000 <score>=0.779599659284\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 683.4499835968018, \"sum\": 683.4499835968018, \"min\": 683.4499835968018}}, \"EndTime\": 1535515916.561454, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515915.876215}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #progress_metric: host=algo-1, completed 99 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 9010, \"sum\": 9010.0, \"min\": 9010}, \"Total Records Seen\": {\"count\": 1, \"max\": 8967430, \"sum\": 8967430.0, \"min\": 8967430}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 100, \"sum\": 100.0, \"min\": 100}}, \"EndTime\": 1535515916.561666, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 98}, \"StartTime\": 1535515915.877973}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=132449.072953 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=99, batch=0 train binary_classification_accuracy <score>=0.753\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=99, batch=0 train binary_classification_cross_entropy <loss>=0.50839453125\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:56 INFO 139937213605696] #quality_metric: host=algo-1, epoch=99, batch=0 train binary_f_1.000 <score>=0.778871978514\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=99, train binary_classification_accuracy <score>=0.749879120879\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=99, train binary_classification_cross_entropy <loss>=0.513281436124\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, epoch=99, train binary_f_1.000 <score>=0.779667579838\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, train binary_classification_accuracy <score>=0.749879120879\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, train binary_classification_cross_entropy <loss>=0.513281436124\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, train binary_f_1.000 <score>=0.779667579838\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"update.time\": {\"count\": 1, \"max\": 711.9228839874268, \"sum\": 711.9228839874268, \"min\": 711.9228839874268}}, \"EndTime\": 1535515917.273967, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515916.561523}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #progress_metric: host=algo-1, completed 100 % of epochs\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 91, \"sum\": 91.0, \"min\": 91}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Total Batches Seen\": {\"count\": 1, \"max\": 9101, \"sum\": 9101.0, \"min\": 9101}, \"Total Records Seen\": {\"count\": 1, \"max\": 9058000, \"sum\": 9058000.0, \"min\": 9058000}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 90570, \"sum\": 90570.0, \"min\": 90570}, \"Reset Count\": {\"count\": 1, \"max\": 101, \"sum\": 101.0, \"min\": 101}}, \"EndTime\": 1535515917.274173, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"training_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\", \"epoch\": 99}, \"StartTime\": 1535515916.562009}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #throughput_metric: host=algo-1, train throughput=127154.53439 records/second\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 WARNING 139937213605696] wait_for_all_workers will not sync workers since the kv store is not running distributed\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] Pulling entire model from kvstore to finalize\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"finalize.time\": {\"count\": 1, \"max\": 2.2430419921875, \"sum\": 2.2430419921875, \"min\": 2.2430419921875}}, \"EndTime\": 1535515917.276694, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515917.274031}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] Saved checkpoint to \"/tmp/tmp9_oCS7/state-0001.params\"\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"Max Batches Seen Between Resets\": {\"count\": 1, \"max\": 10, \"sum\": 10.0, \"min\": 10}, \"Number of Batches Since Last Reset\": {\"count\": 1, \"max\": 10, \"sum\": 10.0, \"min\": 10}, \"Number of Records Since Last Reset\": {\"count\": 1, \"max\": 9430, \"sum\": 9430.0, \"min\": 9430}, \"Total Batches Seen\": {\"count\": 1, \"max\": 10, \"sum\": 10.0, \"min\": 10}, \"Total Records Seen\": {\"count\": 1, \"max\": 9430, \"sum\": 9430.0, \"min\": 9430}, \"Max Records Seen Between Resets\": {\"count\": 1, \"max\": 9430, \"sum\": 9430.0, \"min\": 9430}, \"Reset Count\": {\"count\": 1, \"max\": 1, \"sum\": 1.0, \"min\": 1}}, \"EndTime\": 1535515917.32736, \"Dimensions\": {\"Host\": \"algo-1\", \"Meta\": \"test_data_iter\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515917.282473}\n",
      "\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #test_score (algo-1) : ('binary_classification_accuracy', 0.68748674443266167)\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #test_score (algo-1) : ('binary_classification_cross_entropy', 0.58208658616843689)\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #test_score (algo-1) : ('binary_f_1.000', 0.7211128986467303)\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, test binary_classification_accuracy <score>=0.687486744433\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, test binary_classification_cross_entropy <loss>=0.582086586168\u001b[0m\n",
      "\u001b[31m[08/29/2018 04:11:57 INFO 139937213605696] #quality_metric: host=algo-1, test binary_f_1.000 <score>=0.721112898647\u001b[0m\n",
      "\u001b[31m#metrics {\"Metrics\": {\"totaltime\": {\"count\": 1, \"max\": 74154.75702285767, \"sum\": 74154.75702285767, \"min\": 74154.75702285767}, \"setuptime\": {\"count\": 1, \"max\": 36.03005409240723, \"sum\": 36.03005409240723, \"min\": 36.03005409240723}}, \"EndTime\": 1535515917.329307, \"Dimensions\": {\"Host\": \"algo-1\", \"Operation\": \"training\", \"Algorithm\": \"factorization-machines\"}, \"StartTime\": 1535515917.276765}\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Billable seconds: 172\n"
     ]
    }
   ],
   "source": [
    "fm = sagemaker.estimator.Estimator(containers[boto3.Session().region_name],\n",
    "                                   get_execution_role(), \n",
    "                                   train_instance_count=1, \n",
    "                                   train_instance_type='ml.c4.xlarge',\n",
    "                                   output_path=output_prefix,\n",
    "                                   sagemaker_session=sagemaker.Session())\n",
    "\n",
    "fm.set_hyperparameters(feature_dim=nbFeatures,\n",
    "                      predictor_type='binary_classifier',\n",
    "                      mini_batch_size=1000,\n",
    "                      num_factors=64,\n",
    "                      epochs=100)\n",
    "\n",
    "fm.fit({'train': train_data, 'test': test_data})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deploy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Creating model with name: factorization-machines-2018-08-29-04-12-33-551\n",
      "INFO:sagemaker:Creating endpoint with name factorization-machines-2018-08-29-04-07-20-066\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------!"
     ]
    }
   ],
   "source": [
    "fm_predictor = fm.deploy(instance_type='ml.c4.xlarge', initial_instance_count=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fm_serializer(data):\n",
    "    js = {'instances': []}\n",
    "    for row in data:\n",
    "        js['instances'].append({'features': row.tolist()})\n",
    "    #print js\n",
    "    return json.dumps(js)\n",
    "\n",
    "fm_predictor.content_type = 'application/json'\n",
    "fm_predictor.serializer = fm_serializer\n",
    "fm_predictor.deserializer = json_deserializer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 100)\t1.0\n",
      "  (0, 1164)\t1.0\n",
      "  (1, 100)\t1.0\n",
      "  (1, 1194)\t1.0\n",
      "  (2, 100)\t1.0\n",
      "  (2, 1223)\t1.0\n",
      "  (3, 100)\t1.0\n",
      "  (3, 1224)\t1.0\n",
      "  (4, 100)\t1.0\n",
      "  (4, 1246)\t1.0\n",
      "  (5, 100)\t1.0\n",
      "  (5, 1311)\t1.0\n",
      "  (6, 100)\t1.0\n",
      "  (6, 1347)\t1.0\n",
      "  (7, 100)\t1.0\n",
      "  (7, 1413)\t1.0\n",
      "  (8, 100)\t1.0\n",
      "  (8, 1538)\t1.0\n",
      "  (9, 100)\t1.0\n",
      "  (9, 1771)\t1.0\n"
     ]
    }
   ],
   "source": [
    "print(X_test[1000:1010])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'predictions': [{u'score': 0.2745290696620941, u'predicted_label': 0.0}, {u'score': 0.4928366541862488, u'predicted_label': 0.0}, {u'score': 0.18388329446315765, u'predicted_label': 0.0}, {u'score': 0.523980438709259, u'predicted_label': 1.0}, {u'score': 0.2758810222148895, u'predicted_label': 0.0}, {u'score': 0.32127004861831665, u'predicted_label': 0.0}, {u'score': 0.24220161139965057, u'predicted_label': 0.0}, {u'score': 0.4599955677986145, u'predicted_label': 0.0}, {u'score': 0.1501358151435852, u'predicted_label': 0.0}, {u'score': 0.2602457106113434, u'predicted_label': 0.0}]}\n"
     ]
    }
   ],
   "source": [
    "result = fm_predictor.predict(X_test[2000:2010].toarray())\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 1. 1. 0. 0. 0. 1. 0. 0. 1.]\n"
     ]
    }
   ],
   "source": [
    "print (Y_test[2000:2010])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 200)\t1.0\n",
      "  (0, 1088)\t1.0\n"
     ]
    }
   ],
   "source": [
    "print(X_test[2000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'predictions': [{u'score': 0.2745290696620941, u'predicted_label': 0.0}]}\n",
      "[0.]\n"
     ]
    }
   ],
   "source": [
    "result = fm_predictor.predict(X_test[2000:2001].toarray())\n",
    "print(result)\n",
    "print(Y_test[2000:2001])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python2",
   "language": "python",
   "name": "conda_python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}