notebooks/end-to-end-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# End to end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from time import gmtime, strftime\n",
    "\n",
    "sagemaker_session = sagemaker.Session()\n",
    "role = sagemaker.get_execution_role()\n",
    "bucket = sagemaker_session.default_bucket()\n",
    "\n",
    "timestamp_prefix = strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n",
    "\n",
    "prefix = 'sagemaker/spark-preprocess-demo/' + timestamp_prefix\n",
    "input_prefix = prefix + '/input/raw/abalone'\n",
    "input_preprocessed_prefix = prefix + '/input/preprocessed/abalone'\n",
    "model_prefix = prefix + '/model'\n",
    "#Jay Change\n",
    "mleap_model_prefix = prefix + '/mleap-model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-06-11 18:47:41--  https://s3-us-west-2.amazonaws.com/sparkml-mleap/data/abalone/abalone.csv\n",
      "Resolving s3-us-west-2.amazonaws.com (s3-us-west-2.amazonaws.com)... 52.218.200.240\n",
      "Connecting to s3-us-west-2.amazonaws.com (s3-us-west-2.amazonaws.com)|52.218.200.240|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 191873 (187K) [binary/octet-stream]\n",
      "Saving to: ‘abalone.csv.13’\n",
      "\n",
      "abalone.csv.13      100%[===================>] 187.38K   766KB/s    in 0.2s    \n",
      "\n",
      "2020-06-11 18:47:42 (766 KB/s) - ‘abalone.csv.13’ saved [191873/191873]\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-us-east-1-115731758279/sagemaker/spark-preprocess-demo/2020-06-11-18-47-41/input/raw/abalone/abalone.csv'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fetch the dataset from the SageMaker bucket\n",
    "!wget https://s3-us-west-2.amazonaws.com/sparkml-mleap/data/abalone/abalone.csv\n",
    "\n",
    "# Uploading the training data to S3\n",
    "sagemaker_session.upload_data(path='abalone.csv', bucket=bucket, key_prefix=input_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/SageMaker/SageMaker-Processing-Pipeline/end-to-end-pipeline/container\n",
      "Sending build context to Docker daemon  17.36MB\n",
      "Step 1/34 : FROM openjdk:8-jre-slim\n",
      " ---> 73c63778326a\n",
      "Step 2/34 : RUN apt-get update\n",
      " ---> Using cache\n",
      " ---> 29f4af6e25f0\n",
      "Step 3/34 : RUN apt-get install -y curl unzip python3 python3-setuptools python3-pip python-dev python3-dev python-psutil\n",
      " ---> Using cache\n",
      " ---> f35d5c478797\n",
      "Step 4/34 : RUN pip3 install py4j psutil==5.6.5 mleap==0.8.1 boto3\n",
      " ---> Using cache\n",
      " ---> 7b3f2af45c9a\n",
      "Step 5/34 : RUN apt-get clean\n",
      " ---> Using cache\n",
      " ---> 181918d09ffc\n",
      "Step 6/34 : RUN rm -rf /var/lib/apt/lists/*\n",
      " ---> Using cache\n",
      " ---> 6bc665de5740\n",
      "Step 7/34 : ENV PYTHONHASHSEED 0\n",
      " ---> Using cache\n",
      " ---> 03462d22ec80\n",
      "Step 8/34 : ENV PYTHONIOENCODING UTF-8\n",
      " ---> Using cache\n",
      " ---> bb25aea46ed2\n",
      "Step 9/34 : ENV PIP_DISABLE_PIP_VERSION_CHECK 1\n",
      " ---> Using cache\n",
      " ---> 45eba8760e7c\n",
      "Step 10/34 : ENV HADOOP_VERSION 3.0.0\n",
      " ---> Using cache\n",
      " ---> 2d86b2b22615\n",
      "Step 11/34 : ENV HADOOP_HOME /usr/hadoop-$HADOOP_VERSION\n",
      " ---> Using cache\n",
      " ---> 2d08d5ed0a3e\n",
      "Step 12/34 : ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop\n",
      " ---> Using cache\n",
      " ---> 98611ffa47ba\n",
      "Step 13/34 : ENV PATH $PATH:$HADOOP_HOME/bin\n",
      " ---> Using cache\n",
      " ---> 402733479807\n",
      "Step 14/34 : RUN curl -sL --retry 3   \"http://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz\"   | gunzip   | tar -x -C /usr/  && rm -rf $HADOOP_HOME/share/doc  && chown -R root:root $HADOOP_HOME\n",
      " ---> Using cache\n",
      " ---> 7e6933861fd2\n",
      "Step 15/34 : ENV SPARK_VERSION 2.2.0\n",
      " ---> Using cache\n",
      " ---> 40fc0b3c7087\n",
      "Step 16/34 : ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-without-hadoop\n",
      " ---> Using cache\n",
      " ---> f993b05eb4d1\n",
      "Step 17/34 : ENV SPARK_HOME /usr/spark-${SPARK_VERSION}\n",
      " ---> Using cache\n",
      " ---> d25f37629ab1\n",
      "Step 18/34 : ENV SPARK_DIST_CLASSPATH=\"$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*:$SPARK_HOME/input_custom_jars/*\"\n",
      " ---> Using cache\n",
      " ---> de6b59adbf0f\n",
      "Step 19/34 : ENV PATH $PATH:${SPARK_HOME}/bin\n",
      " ---> Using cache\n",
      " ---> ccb344f8306f\n",
      "Step 20/34 : RUN curl -sL --retry 3   \"https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_PACKAGE}.tgz\"   | gunzip   | tar x -C /usr/  && mv /usr/$SPARK_PACKAGE $SPARK_HOME  && chown -R root:root $SPARK_HOME\n",
      " ---> Using cache\n",
      " ---> 6dd71caf56e7\n",
      "Step 21/34 : ENV PYSPARK_PYTHON=/usr/bin/python3\n",
      " ---> Using cache\n",
      " ---> 2117ac0c0f62\n",
      "Step 22/34 : ENV PATH=\"/usr/bin:/opt/program:${PATH}\"\n",
      " ---> Using cache\n",
      " ---> 3089c24fcb8c\n",
      "Step 23/34 : ENV YARN_RESOURCEMANAGER_USER=\"root\"\n",
      " ---> Using cache\n",
      " ---> b921081ca22c\n",
      "Step 24/34 : ENV YARN_NODEMANAGER_USER=\"root\"\n",
      " ---> Using cache\n",
      " ---> ffeb09e78626\n",
      "Step 25/34 : ENV HDFS_NAMENODE_USER=\"root\"\n",
      " ---> Using cache\n",
      " ---> 3a28c4a99e8e\n",
      "Step 26/34 : ENV HDFS_DATANODE_USER=\"root\"\n",
      " ---> Using cache\n",
      " ---> 3f1a7141a2dc\n",
      "Step 27/34 : ENV HDFS_SECONDARYNAMENODE_USER=\"root\"\n",
      " ---> Using cache\n",
      " ---> 1b55ee6db442\n",
      "Step 28/34 : COPY program /opt/program\n",
      " ---> Using cache\n",
      " ---> 21d9328aa743\n",
      "Step 29/34 : RUN chmod +x /opt/program/submit\n",
      " ---> Using cache\n",
      " ---> 64384c580b4a\n",
      "Step 30/34 : COPY hadoop-config /opt/hadoop-config\n",
      " ---> Using cache\n",
      " ---> 97155d652790\n",
      "Step 31/34 : COPY mleap_spark_assembly.jar $SPARK_HOME/\n",
      " ---> Using cache\n",
      " ---> 4458456a803f\n",
      "Step 32/34 : ENV SPARK_DIST_CLASSPATH=\"$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*:$SPARK_HOME/*\"\n",
      " ---> Using cache\n",
      " ---> d67edc7a929a\n",
      "Step 33/34 : WORKDIR $SPARK_HOME\n",
      " ---> Using cache\n",
      " ---> 7408e68ccfee\n",
      "Step 34/34 : ENTRYPOINT [\"/opt/program/submit\"]\n",
      " ---> Using cache\n",
      " ---> 8426c0e82dfb\n",
      "Successfully built 8426c0e82dfb\n",
      "Successfully tagged sagemaker-spark-example:latest\n",
      "/home/ec2-user/SageMaker/SageMaker-Processing-Pipeline/end-to-end-pipeline\n"
     ]
    }
   ],
   "source": [
    "%cd container\n",
    "!docker build -t sagemaker-spark-example .\n",
    "%cd ../"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING! Using --password via the CLI is insecure. Use --password-stdin.\n",
      "WARNING! Your password will be stored unencrypted in /home/ec2-user/.docker/config.json.\n",
      "Configure a credential helper to remove this warning. See\n",
      "https://docs.docker.com/engine/reference/commandline/login/#credentials-store\n",
      "\n",
      "Login Succeeded\n",
      "\n",
      "An error occurred (RepositoryAlreadyExistsException) when calling the CreateRepository operation: The repository with name 'sagemaker-spark-example' already exists in the registry with id '115731758279'\n",
      "The push refers to repository [115731758279.dkr.ecr.us-east-1.amazonaws.com/sagemaker-spark-example]\n",
      "\n",
      "\u001b[1B2bc255a4: Preparing \n",
      "\u001b[1Bd149d9f9: Preparing \n",
      "\u001b[1B80d41ded: Preparing \n",
      "\u001b[1B2d14254b: Preparing \n",
      "\u001b[1Bfb310c6f: Preparing \n",
      "\u001b[1Be4ad5bcc: Preparing \n",
      "\u001b[1B9f97b723: Preparing \n",
      "\u001b[1B16fa6a66: Preparing \n",
      "\u001b[1Bfd787cef: Preparing \n",
      "\u001b[1Bd64b1aad: Preparing \n",
      "\u001b[1Bb10ea4a6: Preparing \n",
      "\u001b[1B760baedf: Preparing \n",
      "\u001b[1B3663cf66: Preparing \n",
      "\u001b[1B29cec5e1: Preparing \n",
      "\u001b[1B14c2acd3: Layer already exists \u001b[15A\u001b[1K\u001b[K\u001b[8A\u001b[1K\u001b[K\u001b[3A\u001b[1K\u001b[Klatest: digest: sha256:bfe1d2dcc730111c43d4afd8a93e59d9bbff64bae4b79bca8ed346185e890990 size: 3474\n"
     ]
    }
   ],
   "source": [
    "import boto3\n",
    "account_id = boto3.client('sts').get_caller_identity().get('Account')\n",
    "region = boto3.session.Session().region_name\n",
    "\n",
    "ecr_repository = 'sagemaker-spark-example'\n",
    "tag = ':latest'\n",
    "uri_suffix = 'amazonaws.com'\n",
    "if region in ['cn-north-1', 'cn-northwest-1']:\n",
    "    uri_suffix = 'amazonaws.com.cn'\n",
    "spark_repository_uri = '{}.dkr.ecr.{}.{}/{}'.format(account_id, region, uri_suffix, ecr_repository + tag)\n",
    "\n",
    "# Create ECR repository and push docker image\n",
    "!$(aws ecr get-login --region $region --registry-ids $account_id --no-include-email)\n",
    "!aws ecr create-repository --repository-name $ecr_repository\n",
    "!docker tag {ecr_repository + tag} $spark_repository_uri\n",
    "!docker push $spark_repository_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting preprocess.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile preprocess.py\n",
    "from __future__ import print_function\n",
    "from __future__ import unicode_literals\n",
    "\n",
    "import time\n",
    "import sys\n",
    "import os\n",
    "import shutil\n",
    "import csv\n",
    "\n",
    "import boto3\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.sql.types import StructField, StructType, StringType, DoubleType\n",
    "from pyspark.ml.feature import StringIndexer, VectorIndexer, OneHotEncoder, VectorAssembler\n",
    "from pyspark.sql.functions import *\n",
    "from mleap.pyspark.spark_support import SimpleSparkSerializer\n",
    "\n",
    "def csv_line(data):\n",
    "    r = ','.join(str(d) for d in data[1])\n",
    "    return str(data[0]) + \",\" + r\n",
    "\n",
    "\n",
    "def main():\n",
    "    spark = SparkSession.builder.appName(\"PySparkAbalone\").getOrCreate()\n",
    "    \n",
    "    # Convert command line args into a map of args\n",
    "    args_iter = iter(sys.argv[1:])\n",
    "    args = dict(zip(args_iter, args_iter))\n",
    "    \n",
    "    # This is needed to save RDDs which is the only way to write nested Dataframes into CSV format\n",
    "    spark.sparkContext._jsc.hadoopConfiguration().set(\"mapred.output.committer.class\",\n",
    "                                                      \"org.apache.hadoop.mapred.FileOutputCommitter\")\n",
    "    \n",
    "    # Defining the schema corresponding to the input data. The input data does not contain the headers\n",
    "    schema = StructType([StructField(\"sex\", StringType(), True), \n",
    "                         StructField(\"length\", DoubleType(), True),\n",
    "                         StructField(\"diameter\", DoubleType(), True),\n",
    "                         StructField(\"height\", DoubleType(), True),\n",
    "                         StructField(\"whole_weight\", DoubleType(), True),\n",
    "                         StructField(\"shucked_weight\", DoubleType(), True),\n",
    "                         StructField(\"viscera_weight\", DoubleType(), True), \n",
    "                         StructField(\"shell_weight\", DoubleType(), True), \n",
    "                         StructField(\"rings\", DoubleType(), True)])\n",
    "\n",
    "    # Downloading the data from S3 into a Dataframe\n",
    "    total_df = spark.read.csv(('s3a://' + os.path.join(args['s3_input_bucket'], args['s3_input_key_prefix'],\n",
    "                                                   'abalone.csv')), header=False, schema=schema)\n",
    "\n",
    "    #StringIndexer on the sex column which has categorical value\n",
    "    sex_indexer = StringIndexer(inputCol=\"sex\", outputCol=\"indexed_sex\")\n",
    "    \n",
    "    #one-hot-encoding is being performed on the string-indexed sex column (indexed_sex)\n",
    "    sex_encoder = OneHotEncoder(inputCol=\"indexed_sex\", outputCol=\"sex_vec\")\n",
    "\n",
    "    #vector-assembler will bring all the features to a 1D vector for us to save easily into CSV format\n",
    "    assembler = VectorAssembler(inputCols=[\"sex_vec\", \n",
    "                                           \"length\", \n",
    "                                           \"diameter\", \n",
    "                                           \"height\", \n",
    "                                           \"whole_weight\", \n",
    "                                           \"shucked_weight\", \n",
    "                                           \"viscera_weight\", \n",
    "                                           \"shell_weight\"], \n",
    "                                outputCol=\"features\")\n",
    "    \n",
    "    # The pipeline comprises of the steps added above\n",
    "    pipeline = Pipeline(stages=[sex_indexer, sex_encoder, assembler])\n",
    "    \n",
    "    # This step trains the feature transformers\n",
    "    model = pipeline.fit(total_df)\n",
    "    \n",
    "    # This step transforms the dataset with information obtained from the previous fit\n",
    "    transformed_total_df = model.transform(total_df)\n",
    "    \n",
    "    # Split the overall dataset into 80-20 training and validation\n",
    "    (train_df, validation_df) = transformed_total_df.randomSplit([0.8, 0.2])\n",
    "    \n",
    "    # Convert the train dataframe to RDD to save in CSV format and upload to S3\n",
    "    train_rdd = train_df.rdd.map(lambda x: (x.rings, x.features))\n",
    "    train_lines = train_rdd.map(csv_line)\n",
    "    train_lines.saveAsTextFile('s3a://' + os.path.join(args['s3_output_bucket'], args['s3_output_key_prefix'], 'train'))\n",
    "    \n",
    "    # Convert the validation dataframe to RDD to save in CSV format and upload to S3\n",
    "    validation_rdd = validation_df.rdd.map(lambda x: (x.rings, x.features))\n",
    "    validation_lines = validation_rdd.map(csv_line)\n",
    "    validation_lines.saveAsTextFile('s3a://' + os.path.join(args['s3_output_bucket'], args['s3_output_key_prefix'], 'validation'))\n",
    "    \n",
    "    # Serialize and store the model via MLeap  \n",
    "    SimpleSparkSerializer().serializeToBundle(model, \"jar:file:/opt/ml/model.zip\", validation_df)    \n",
    "    # Unzip the model as SageMaker expects a .tar.gz file but MLeap produces a .zip file\n",
    "    import zipfile\n",
    "    with zipfile.ZipFile(\"/opt/ml/model.zip\") as zf:\n",
    "        zf.extractall(\"/opt/ml/model\")\n",
    "\n",
    "    # Writw back the content as a .tar.gz file\n",
    "    import tarfile\n",
    "    with tarfile.open(\"/opt/ml/model.tar.gz\", \"w:gz\") as tar:\n",
    "        tar.add(\"/opt/ml/model/bundle.json\", arcname='bundle.json')\n",
    "        tar.add(\"/opt/ml/model/root\", arcname='root')\n",
    "    \n",
    "    # Upload the model in tar.gz format to S3 so that it can be used with SageMaker for inference later\n",
    "    s3 = boto3.resource('s3') \n",
    "    file_name = os.path.join(args['s3_mleap_model_prefix'], 'model.tar.gz')\n",
    "    s3.Bucket(args['s3_model_bucket']).upload_file('/opt/ml/model.tar.gz', file_name)    \n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Job Name:  spark-preprocessor-2020-06-11-18-47-45-439\n",
      "Inputs:  [{'InputName': 'code', 'S3Input': {'S3Uri': 's3://sagemaker-us-east-1-115731758279/spark-preprocessor-2020-06-11-18-47-45-439/input/code/preprocess.py', 'LocalPath': '/opt/ml/processing/input/code', 'S3DataType': 'S3Prefix', 'S3InputMode': 'File', 'S3DataDistributionType': 'FullyReplicated', 'S3CompressionType': 'None'}}]\n",
      "Outputs:  []\n",
      ".......................\u001b[34mWARNING: /usr/hadoop-3.0.0/logs does not exist. Creating.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:20,774 INFO namenode.NameNode: STARTUP_MSG: \u001b[0m\n",
      "\u001b[34m/************************************************************\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG: Starting NameNode\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG:   host = algo-1/10.0.246.193\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG:   args = [-format, -force]\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG:   version = 3.0.0\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG:   classpath = /usr/hadoop-3.0.0/etc/hadoop:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-beanutils-1.9.3.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/woodstox-core-5.0.3.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/nimbus-jose-jwt-4.41.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-identity-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/log4j-1.2.17.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-databind-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/curator-client-2.12.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/hamcrest-core-1.3.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-cli-1.2.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerby-config-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/httpclient-4.5.2.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/guava-11.0.2.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-io-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jul-to-slf4j-1.7.25.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-core-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-lang-2.6.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-client-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-util-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/paranamer-2.3.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/httpcore-4.4.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-server-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jersey-servlet-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-jaxrs-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/zookeeper-3.4.9.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/curator-recipes-2.12.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-math3-3.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-core-asl-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/snappy-java-1.0.5.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/mockito-all-1.8.5.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-security-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jersey-json-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jsch-0.1.54.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jaxb-api-2.2.11.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-servlet-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jersey-core-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/netty-3.10.5.Final.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-common-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/protobuf-java-2.5.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-xml-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jsp-api-2.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/re2j-1.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerby-asn1-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-mapper-asl-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-server-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-codec-1.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-crypto-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-lang3-3.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jaxb-impl-2.2.3-1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jsr305-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jsr311-api-1.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-io-2.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-simplekdc-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/slf4j-api-1.7.25.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-xc-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/asm-5.0.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/hadoop-auth-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/gson-2.2.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/xz-1.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/json-smart-2.3.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jettison-1.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-logging-1.1.3.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/hadoop-annotations-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jersey-server-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/junit-4.11.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerby-xdr-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-compress-1.4.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/avro-1.7.7.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerby-pkix-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/curator-framework-2.12.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/token-provider-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-admin-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-configuration2-2.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/htrace-core4-4.1.0-incubating.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-webapp-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jetty-http-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-net-3.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/metrics-core-3.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerby-util-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-core-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jcip-annotations-1.0-1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/accessors-smart-1.2.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/javax.servlet-api-3.1.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/kerb-util-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/commons-collections-3.2.2.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/jackson-annotations-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/stax2-api-3.1.4.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/aws-java-sdk-bundle-1.11.199.jar:/usr/hadoop-3.0.0/share/hadoop/common/lib/hadoop-aws-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/hadoop-nfs-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/hadoop-kms-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/common/hadoop-common-3.0.0-tests.jar:/usr/hadoop-3.0.0/share/hadoop/common/hadoop-common-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/okhttp-2.4.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-beanutils-1.9.3.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/woodstox-core-5.0.3.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/nimbus-jose-jwt-4.41.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-identity-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/log4j-1.2.17.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-databind-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/curator-client-2.12.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-cli-1.2.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerby-config-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/httpclient-4.5.2.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/guava-11.0.2.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-io-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-core-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-lang-2.6.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-client-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-util-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/paranamer-2.3.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/httpcore-4.4.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-server-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jersey-servlet-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-jaxrs-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/zookeeper-3.4.9.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/netty-all-4.0.23.Final.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/curator-recipes-2.12.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-math3-3.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-core-asl-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/snappy-java-1.0.5.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-security-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jersey-json-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jsch-0.1.54.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jaxb-api-2.2.11.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-servlet-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jersey-core-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/netty-3.10.5.Final.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-common-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/protobuf-java-2.5.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-xml-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/leveldbjni-all-1.8.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/re2j-1.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerby-asn1-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-mapper-asl-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-server-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-codec-1.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-crypto-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-lang3-3.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jaxb-impl-2.2.3-1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jsr305-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jsr311-api-1.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-io-2.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-simplekdc-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-xc-1.9.13.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/asm-5.0.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/hadoop-auth-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/gson-2.2.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/xz-1.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/json-smart-2.3.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jettison-1.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-logging-1.1.3.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/hadoop-annotations-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jersey-server-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-util-ajax-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerby-xdr-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-compress-1.4.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/avro-1.7.7.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerby-pkix-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/curator-framework-2.12.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/token-provider-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-admin-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/json-simple-1.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-configuration2-2.1.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/htrace-core4-4.1.0-incubating.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-webapp-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jetty-http-9.3.19.v20170502.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-net-3.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerby-util-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-core-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jcip-annotations-1.0-1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/accessors-smart-1.2.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/javax.servlet-api-3.1.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/kerb-util-1.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-collections-3.2.2.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/jackson-annotations-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/stax2-api-3.1.4.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/commons-daemon-1.0.13.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/lib/okio-1.4.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-nfs-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-client-3.0.0-tests.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-httpfs-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-client-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-native-client-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-native-client-3.0.0-tests.jar:/usr/hadoop-3.0.0/share/hadoop/hdfs/hadoop-hdfs-3.0.0-tests.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.0.0-tests.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-plugins-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-common-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-app-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-shuffle-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-nativetask-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-core-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/mapreduce/hadoop-mapreduce-client-hs-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/fst-2.50.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jsp-api-2.1-6.1.14.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/commons-csv-1.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jasper-runtime-5.5.23.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/geronimo-jcache_1.0_spec-1.0-alpha-1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-annotations-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/guice-4.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jamon-runtime-2.4.1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/htrace-core-3.1.0-incubating.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-procedure-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/servlet-api-2.5-6.1.14.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/guice-servlet-4.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jasper-compiler-5.5.23.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/java-util-1.9.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/findbugs-annotations-1.3.9-1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-protocol-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jersey-client-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-client-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/javax.inject-1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jackson-jaxrs-base-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/commons-math-2.2.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jackson-module-jaxb-annotations-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-hadoop-compat-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jcodings-1.0.8.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/joni-2.1.2.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jsp-2.1-6.1.14.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/ehcache-3.3.1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/json-io-2.5.1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-prefix-tree-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-hadoop2-compat-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/mssql-jdbc-6.2.1.jre7.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/disruptor-3.3.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/metrics-core-2.2.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jersey-guice-1.19.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/commons-el-1.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/jackson-jaxrs-json-provider-2.7.8.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/commons-httpclient-3.1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/HikariCP-java7-2.4.12.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-server-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/hbase-common-1.2.6.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/metrics-core-3.0.1.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/lib/aopalliance-1.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-api-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-common-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-client-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-timeline-pluginstorage-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-applicationhistoryservice-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-tests-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-registry-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-nodemanager-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-applications-dist\u001b[0m\n",
      "\u001b[34mributedshell-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-timelineservice-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-router-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-applications-unmanaged-am-launcher-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-sharedcachemanager-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-timelineservice-hbase-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-web-proxy-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-common-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-timelineservice-hbase-tests-3.0.0.jar:/usr/hadoop-3.0.0/share/hadoop/yarn/hadoop-yarn-server-resourcemanager-3.0.0.jar\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG:   build = https://git-wip-us.apache.org/repos/asf/hadoop.git -r c25427ceca461ee979d30edd7a4b0f50718e6533; compiled by 'andrew' on 2017-12-08T19:16Z\u001b[0m\n",
      "\u001b[34mSTARTUP_MSG:   java = 1.8.0_252\u001b[0m\n",
      "\u001b[34m************************************************************/\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:20,782 INFO namenode.NameNode: registered UNIX signal handlers for [TERM, HUP, INT]\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:20,786 INFO namenode.NameNode: createNameNode [-format, -force]\u001b[0m\n",
      "\u001b[34mFormatting using clusterid: CID-05f92a92-963a-48de-8508-bf3e224c1d29\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,237 INFO namenode.FSEditLog: Edit logging is async:true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,247 INFO namenode.FSNamesystem: KeyProvider: null\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,248 INFO namenode.FSNamesystem: fsLock is fair: true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,250 INFO namenode.FSNamesystem: Detailed lock hold time metrics enabled: false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,254 INFO namenode.FSNamesystem: fsOwner             = root (auth:SIMPLE)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,255 INFO namenode.FSNamesystem: supergroup          = supergroup\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,255 INFO namenode.FSNamesystem: isPermissionEnabled = true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,255 INFO namenode.FSNamesystem: HA Enabled: false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,288 INFO common.Util: dfs.datanode.fileio.profiling.sampling.percentage set to 0. Disabling file IO profiling\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,300 INFO blockmanagement.DatanodeManager: dfs.block.invalidate.limit: configured=1000, counted=60, effected=1000\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,300 INFO blockmanagement.DatanodeManager: dfs.namenode.datanode.registration.ip-hostname-check=true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,304 INFO blockmanagement.BlockManager: dfs.namenode.startup.delay.block.deletion.sec is set to 000:00:00:00.000\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,304 INFO blockmanagement.BlockManager: The block deletion will start around 2020 Jun 11 18:51:21\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,306 INFO util.GSet: Computing capacity for map BlocksMap\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,306 INFO util.GSet: VM type       = 64-bit\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,307 INFO util.GSet: 2.0% max memory 6.7 GB = 136.4 MB\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,307 INFO util.GSet: capacity      = 2^24 = 16777216 entries\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,361 INFO blockmanagement.BlockManager: dfs.block.access.token.enable = false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,365 INFO Configuration.deprecation: No unit for dfs.namenode.safemode.extension(30000) assuming MILLISECONDS\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,365 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.threshold-pct = 0.9990000128746033\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,365 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.min.datanodes = 0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,365 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.extension = 30000\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: defaultReplication         = 3\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: maxReplication             = 512\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: minReplication             = 1\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: maxReplicationStreams      = 2\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: redundancyRecheckInterval  = 3000ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: encryptDataTransfer        = false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,366 INFO blockmanagement.BlockManager: maxNumBlocksToLog          = 1000\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,397 INFO util.GSet: Computing capacity for map INodeMap\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,397 INFO util.GSet: VM type       = 64-bit\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,397 INFO util.GSet: 1.0% max memory 6.7 GB = 68.2 MB\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,397 INFO util.GSet: capacity      = 2^23 = 8388608 entries\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,414 INFO namenode.FSDirectory: ACLs enabled? false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,414 INFO namenode.FSDirectory: POSIX ACL inheritance enabled? true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,414 INFO namenode.FSDirectory: XAttrs enabled? true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,414 INFO namenode.NameNode: Caching file names occurring more than 10 times\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,418 INFO snapshot.SnapshotManager: Loaded config captureOpenFiles: false, skipCaptureAccessTimeOnlyChange: false, snapshotDiffAllowSnapRootDescendant: true\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,421 INFO util.GSet: Computing capacity for map cachedBlocks\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,421 INFO util.GSet: VM type       = 64-bit\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,422 INFO util.GSet: 0.25% max memory 6.7 GB = 17.0 MB\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,422 INFO util.GSet: capacity      = 2^21 = 2097152 entries\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,428 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.window.num.buckets = 10\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,428 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.num.users = 10\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,428 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.windows.minutes = 1,5,25\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,431 INFO namenode.FSNamesystem: Retry cache on namenode is enabled\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,431 INFO namenode.FSNamesystem: Retry cache will use 0.03 of total heap and retry cache entry expiry time is 600000 millis\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,432 INFO util.GSet: Computing capacity for map NameNodeRetryCache\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,432 INFO util.GSet: VM type       = 64-bit\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,433 INFO util.GSet: 0.029999999329447746% max memory 6.7 GB = 2.0 MB\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,433 INFO util.GSet: capacity      = 2^18 = 262144 entries\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,453 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1358730645-10.0.246.193-1591901481448\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,465 INFO common.Storage: Storage directory /opt/amazon/hadoop/hdfs/namenode has been successfully formatted.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,476 INFO namenode.FSImageFormatProtobuf: Saving image file /opt/amazon/hadoop/hdfs/namenode/current/fsimage.ckpt_0000000000000000000 using no compression\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,557 INFO namenode.FSImageFormatProtobuf: Image file /opt/amazon/hadoop/hdfs/namenode/current/fsimage.ckpt_0000000000000000000 of size 389 bytes saved in 0 seconds.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,567 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:21,570 INFO namenode.NameNode: SHUTDOWN_MSG: \u001b[0m\n",
      "\u001b[34m/************************************************************\u001b[0m\n",
      "\u001b[34mSHUTDOWN_MSG: Shutting down NameNode at algo-1/10.0.246.193\u001b[0m\n",
      "\u001b[34m************************************************************/\u001b[0m\n",
      "\u001b[34mStarting namenodes on [algo-1]\u001b[0m\n",
      "\u001b[34malgo-1: /usr/hadoop-3.0.0/bin/../libexec/hadoop-functions.sh: line 981: ssh: command not found\u001b[0m\n",
      "\u001b[34mStarting datanodes\u001b[0m\n",
      "\u001b[34mlocalhost: /usr/hadoop-3.0.0/bin/../libexec/hadoop-functions.sh: line 981: ssh: command not found\u001b[0m\n",
      "\u001b[34mStarting secondary namenodes [ip-10-0-246-193.ec2.internal]\u001b[0m\n",
      "\u001b[34mip-10-0-246-193.ec2.internal: /usr/hadoop-3.0.0/bin/../libexec/hadoop-functions.sh: line 981: ssh: command not found\u001b[0m\n",
      "\u001b[35mWARNING: Use of this script to start HDFS daemons is deprecated.\u001b[0m\n",
      "\u001b[35mWARNING: Attempting to execute replacement \"hdfs --daemon start\" instead.\u001b[0m\n",
      "\u001b[35mWARNING: /usr/hadoop-3.0.0/logs does not exist. Creating.\u001b[0m\n",
      "\u001b[35mWARNING: YARN_LOG_DIR has been replaced by HADOOP_LOG_DIR. Using value of YARN_LOG_DIR.\u001b[0m\n",
      "\u001b[35mWARNING: Use of this script to start YARN daemons is deprecated.\u001b[0m\n",
      "\u001b[35mWARNING: Attempting to execute replacement \"yarn --daemon start\" instead.\u001b[0m\n",
      "\u001b[35mWARNING: YARN_LOG_DIR has been replaced by HADOOP_LOG_DIR. Using value of YARN_LOG_DIR.\u001b[0m\n",
      "\u001b[35mWARNING: /var/log/yarn/ does not exist. Creating.\u001b[0m\n",
      "\u001b[34mWARNING: Use of this script to start HDFS daemons is deprecated.\u001b[0m\n",
      "\u001b[34mWARNING: Attempting to execute replacement \"hdfs --daemon start\" instead.\u001b[0m\n",
      "\u001b[34mWARNING: Use of this script to start HDFS daemons is deprecated.\u001b[0m\n",
      "\u001b[34mWARNING: Attempting to execute replacement \"hdfs --daemon start\" instead.\u001b[0m\n",
      "\u001b[34mWARNING: YARN_LOG_DIR has been replaced by HADOOP_LOG_DIR. Using value of YARN_LOG_DIR.\u001b[0m\n",
      "\u001b[34mStarting resourcemanager\u001b[0m\n",
      "\u001b[34mWARNING: YARN_LOG_DIR has been replaced by HADOOP_LOG_DIR. Using value of YARN_LOG_DIR.\u001b[0m\n",
      "\u001b[34mWARNING: /var/log/yarn/ does not exist. Creating.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34mStarting nodemanagers\u001b[0m\n",
      "\u001b[34mWARNING: YARN_LOG_DIR has been replaced by HADOOP_LOG_DIR. Using value of YARN_LOG_DIR.\u001b[0m\n",
      "\u001b[34mlocalhost: /usr/hadoop-3.0.0/bin/../libexec/hadoop-functions.sh: line 981: ssh: command not found\u001b[0m\n",
      "\u001b[34mIvy Default Cache set to: /root/.ivy2/cache\u001b[0m\n",
      "\u001b[34mThe jars for the packages stored in: /root/.ivy2/jars\u001b[0m\n",
      "\u001b[34m:: loading settings :: url = jar:file:/usr/spark-2.2.0/jars/ivy-2.4.0.jar!/org/apache/ivy/core/settings/ivysettings.xml\u001b[0m\n",
      "\u001b[34mml.combust.mleap#mleap-spark_2.11 added as a dependency\u001b[0m\n",
      "\u001b[34m:: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0\u001b[0m\n",
      "\u001b[34m#011confs: [default]\u001b[0m\n",
      "\u001b[34m#011found ml.combust.mleap#mleap-spark_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found ml.combust.mleap#mleap-spark-base_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found ml.combust.mleap#mleap-runtime_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found ml.combust.mleap#mleap-core_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found ml.combust.mleap#mleap-base_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found ml.combust.mleap#mleap-tensor_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found io.spray#spray-json_2.11;1.3.2 in central\u001b[0m\n",
      "\u001b[34m#011found org.apache.spark#spark-mllib-local_2.11;2.2.0 in central\u001b[0m\n",
      "\u001b[34m#011found org.scalanlp#breeze_2.11;0.13.1 in central\u001b[0m\n",
      "\u001b[34m#011found org.scalanlp#breeze-macros_2.11;0.13.1 in central\u001b[0m\n",
      "\u001b[34m#011found org.scala-lang#scala-reflect;2.11.8 in central\u001b[0m\n",
      "\u001b[34m#011found com.github.fommil.netlib#core;1.1.2 in central\u001b[0m\n",
      "\u001b[34m#011found net.sourceforge.f2j#arpack_combined_all;0.1 in central\u001b[0m\n",
      "\u001b[34m#011found net.sf.opencsv#opencsv;2.3 in central\u001b[0m\n",
      "\u001b[34m#011found com.github.rwl#jtransforms;2.4.0 in central\u001b[0m\n",
      "\u001b[34m#011found junit#junit;4.12 in central\u001b[0m\n",
      "\u001b[34m#011found org.hamcrest#hamcrest-core;1.3 in central\u001b[0m\n",
      "\u001b[34m#011found org.spire-math#spire_2.11;0.13.0 in central\u001b[0m\n",
      "\u001b[34m#011found org.spire-math#spire-macros_2.11;0.13.0 in central\u001b[0m\n",
      "\u001b[34m#011found org.typelevel#machinist_2.11;0.6.1 in central\u001b[0m\n",
      "\u001b[34m#011found com.chuusai#shapeless_2.11;2.3.2 in central\u001b[0m\n",
      "\u001b[34m#011found org.typelevel#macro-compat_2.11;1.1.1 in central\u001b[0m\n",
      "\u001b[34m#011found org.slf4j#slf4j-api;1.7.16 in central\u001b[0m\n",
      "\u001b[34m#011found org.apache.commons#commons-math3;3.4.1 in central\u001b[0m\n",
      "\u001b[34m#011found org.apache.spark#spark-tags_2.11;2.2.0 in central\u001b[0m\n",
      "\u001b[34m#011found org.spark-project.spark#unused;1.0.0 in central\u001b[0m\n",
      "\u001b[34m#011found ml.combust.bundle#bundle-ml_2.11;0.8.1 in central\u001b[0m\n",
      "\u001b[34m#011found com.trueaccord.scalapb#scalapb-runtime_2.11;0.6.0 in central\u001b[0m\n",
      "\u001b[34m#011found com.trueaccord.lenses#lenses_2.11;0.4.12 in central\u001b[0m\n",
      "\u001b[34m#011found com.lihaoyi#fastparse_2.11;0.4.2 in central\u001b[0m\n",
      "\u001b[34m#011found com.lihaoyi#fastparse-utils_2.11;0.4.2 in central\u001b[0m\n",
      "\u001b[34m#011found com.lihaoyi#sourcecode_2.11;0.1.3 in central\u001b[0m\n",
      "\u001b[34m#011found com.google.protobuf#protobuf-java;3.3.1 in central\u001b[0m\n",
      "\u001b[34m#011found com.jsuereth#scala-arm_2.11;2.0 in central\u001b[0m\n",
      "\u001b[34m#011found com.typesafe#config;1.3.0 in central\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/mleap/mleap-spark_2.11/0.8.1/mleap-spark_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.mleap#mleap-spark_2.11;0.8.1!mleap-spark_2.11.jar (20ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/mleap/mleap-spark-base_2.11/0.8.1/mleap-spark-base_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.mleap#mleap-spark-base_2.11;0.8.1!mleap-spark-base_2.11.jar (5ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/mleap/mleap-runtime_2.11/0.8.1/mleap-runtime_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.mleap#mleap-runtime_2.11;0.8.1!mleap-runtime_2.11.jar (43ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/mleap/mleap-core_2.11/0.8.1/mleap-core_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.mleap#mleap-core_2.11;0.8.1!mleap-core_2.11.jar (23ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/bundle/bundle-ml_2.11/0.8.1/bundle-ml_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.bundle#bundle-ml_2.11;0.8.1!bundle-ml_2.11.jar (34ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/scala-lang/scala-reflect/2.11.8/scala-reflect-2.11.8.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.scala-lang#scala-reflect;2.11.8!scala-reflect.jar (96ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/mleap/mleap-base_2.11/0.8.1/mleap-base_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.mleap#mleap-base_2.11;0.8.1!mleap-base_2.11.jar (2ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/ml/combust/mleap/mleap-tensor_2.11/0.8.1/mleap-tensor_2.11-0.8.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] ml.combust.mleap#mleap-tensor_2.11;0.8.1!mleap-tensor_2.11.jar (4ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/apache/spark/spark-mllib-local_2.11/2.2.0/spark-mllib-local_2.11-2.2.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.apache.spark#spark-mllib-local_2.11;2.2.0!spark-mllib-local_2.11.jar (8ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/github/rwl/jtransforms/2.4.0/jtransforms-2.4.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.github.rwl#jtransforms;2.4.0!jtransforms.jar (18ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/io/spray/spray-json_2.11/1.3.2/spray-json_2.11-1.3.2.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] io.spray#spray-json_2.11;1.3.2!spray-json_2.11.jar(bundle) (6ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/scalanlp/breeze_2.11/0.13.1/breeze_2.11-0.13.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.scalanlp#breeze_2.11;0.13.1!breeze_2.11.jar (93ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/apache/commons/commons-math3/3.4.1/commons-math3-3.4.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.apache.commons#commons-math3;3.4.1!commons-math3.jar (15ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/apache/spark/spark-tags_2.11/2.2.0/spark-tags_2.11-2.2.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.apache.spark#spark-tags_2.11;2.2.0!spark-tags_2.11.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.spark-project.spark#unused;1.0.0!unused.jar (2ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/scalanlp/breeze-macros_2.11/0.13.1/breeze-macros_2.11-0.13.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.scalanlp#breeze-macros_2.11;0.13.1!breeze-macros_2.11.jar (4ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/github/fommil/netlib/core/1.1.2/core-1.1.2.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.github.fommil.netlib#core;1.1.2!core.jar (4ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/net/sourceforge/f2j/arpack_combined_all/0.1/arpack_combined_all-0.1-javadoc.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] net.sourceforge.f2j#arpack_combined_all;0.1!arpack_combined_all.jar (46ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/net/sf/opencsv/opencsv/2.3/opencsv-2.3.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] net.sf.opencsv#opencsv;2.3!opencsv.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/spire-math/spire_2.11/0.13.0/spire_2.11-0.13.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.spire-math#spire_2.11;0.13.0!spire_2.11.jar (84ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/chuusai/shapeless_2.11/2.3.2/shapeless_2.11-2.3.2.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.chuusai#shapeless_2.11;2.3.2!shapeless_2.11.jar(bundle) (27ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/slf4j/slf4j-api/1.7.16/slf4j-api-1.7.16.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.slf4j#slf4j-api;1.7.16!slf4j-api.jar (2ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] junit#junit;4.12!junit.jar (4ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.hamcrest#hamcrest-core;1.3!hamcrest-core.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/spire-math/spire-macros_2.11/0.13.0/spire-macros_2.11-0.13.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.spire-math#spire-macros_2.11;0.13.0!spire-macros_2.11.jar (2ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/typelevel/machinist_2.11/0.6.1/machinist_2.11-0.6.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.typelevel#machinist_2.11;0.6.1!machinist_2.11.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/org/typelevel/macro-compat_2.11/1.1.1/macro-compat_2.11-1.1.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] org.typelevel#macro-compat_2.11;1.1.1!macro-compat_2.11.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/trueaccord/scalapb/scalapb-runtime_2.11/0.6.0/scalapb-runtime_2.11-0.6.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.trueaccord.scalapb#scalapb-runtime_2.11;0.6.0!scalapb-runtime_2.11.jar (25ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.jsuereth#scala-arm_2.11;2.0!scala-arm_2.11.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/typesafe/config/1.3.0/config-1.3.0.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.typesafe#config;1.3.0!config.jar(bundle) (4ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/trueaccord/lenses/lenses_2.11/0.4.12/lenses_2.11-0.4.12.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.trueaccord.lenses#lenses_2.11;0.4.12!lenses_2.11.jar (3ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/lihaoyi/fastparse_2.11/0.4.2/fastparse_2.11-0.4.2.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.lihaoyi#fastparse_2.11;0.4.2!fastparse_2.11.jar (5ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.3.1/protobuf-java-3.3.1.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.google.protobuf#protobuf-java;3.3.1!protobuf-java.jar(bundle) (14ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/lihaoyi/fastparse-utils_2.11/0.4.2/fastparse-utils_2.11-0.4.2.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.lihaoyi#fastparse-utils_2.11;0.4.2!fastparse-utils_2.11.jar (6ms)\u001b[0m\n",
      "\u001b[34mdownloading https://repo1.maven.org/maven2/com/lihaoyi/sourcecode_2.11/0.1.3/sourcecode_2.11-0.1.3.jar ...\u001b[0m\n",
      "\u001b[34m#011[SUCCESSFUL ] com.lihaoyi#sourcecode_2.11;0.1.3!sourcecode_2.11.jar (4ms)\u001b[0m\n",
      "\u001b[34m:: resolution report :: resolve 3145ms :: artifacts dl 633ms\u001b[0m\n",
      "\u001b[34m#011:: modules in use:\u001b[0m\n",
      "\u001b[34m#011com.chuusai#shapeless_2.11;2.3.2 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.github.fommil.netlib#core;1.1.2 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.github.rwl#jtransforms;2.4.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.google.protobuf#protobuf-java;3.3.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.jsuereth#scala-arm_2.11;2.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.lihaoyi#fastparse-utils_2.11;0.4.2 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.lihaoyi#fastparse_2.11;0.4.2 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.lihaoyi#sourcecode_2.11;0.1.3 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.trueaccord.lenses#lenses_2.11;0.4.12 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.trueaccord.scalapb#scalapb-runtime_2.11;0.6.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011com.typesafe#config;1.3.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011io.spray#spray-json_2.11;1.3.2 from central in [default]\u001b[0m\n",
      "\u001b[34m#011junit#junit;4.12 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.bundle#bundle-ml_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.mleap#mleap-base_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.mleap#mleap-core_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.mleap#mleap-runtime_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.mleap#mleap-spark-base_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.mleap#mleap-spark_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011ml.combust.mleap#mleap-tensor_2.11;0.8.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011net.sf.opencsv#opencsv;2.3 from central in [default]\u001b[0m\n",
      "\u001b[34m#011net.sourceforge.f2j#arpack_combined_all;0.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.apache.commons#commons-math3;3.4.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.apache.spark#spark-mllib-local_2.11;2.2.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.apache.spark#spark-tags_2.11;2.2.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.hamcrest#hamcrest-core;1.3 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.scala-lang#scala-reflect;2.11.8 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.scalanlp#breeze-macros_2.11;0.13.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.scalanlp#breeze_2.11;0.13.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.slf4j#slf4j-api;1.7.16 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.spark-project.spark#unused;1.0.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.spire-math#spire-macros_2.11;0.13.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.spire-math#spire_2.11;0.13.0 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.typelevel#machinist_2.11;0.6.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011org.typelevel#macro-compat_2.11;1.1.1 from central in [default]\u001b[0m\n",
      "\u001b[34m#011---------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m#011|                  |            modules            ||   artifacts   |\u001b[0m\n",
      "\u001b[34m#011|       conf       | number| search|dwnlded|evicted|| number|dwnlded|\u001b[0m\n",
      "\u001b[34m#011---------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m#011|      default     |   35  |   35  |   35  |   0   ||   35  |   35  |\u001b[0m\n",
      "\u001b[34m#011---------------------------------------------------------------------\u001b[0m\n",
      "\u001b[34m:: retrieving :: org.apache.spark#spark-submit-parent\u001b[0m\n",
      "\u001b[34m#011confs: [default]\u001b[0m\n",
      "\u001b[34m#01135 artifacts copied, 0 already retrieved (52376kB/61ms)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,290 INFO spark.SparkContext: Running Spark version 2.2.0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,446 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,534 INFO spark.SparkContext: Submitted application: PySparkAbalone\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,549 INFO spark.SecurityManager: Changing view acls to: root\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,549 INFO spark.SecurityManager: Changing modify acls to: root\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,549 INFO spark.SecurityManager: Changing view acls groups to: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,550 INFO spark.SecurityManager: Changing modify acls groups to: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,550 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(root); groups with view permissions: Set(); users  with modify permissions: Set(root); groups with modify permissions: Set()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,744 INFO util.Utils: Successfully started service 'sparkDriver' on port 33983.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,758 INFO spark.SparkEnv: Registering MapOutputTracker\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,771 INFO spark.SparkEnv: Registering BlockManagerMaster\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,773 INFO storage.BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,773 INFO storage.BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,780 INFO storage.DiskBlockManager: Created local directory at /tmp/blockmgr-2992aa69-39da-45dd-a2bc-d620c2746ceb\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,795 INFO memory.MemoryStore: MemoryStore started with capacity 366.3 MB\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,870 INFO spark.SparkEnv: Registering OutputCommitCoordinator\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,928 INFO util.log: Logging initialized @5892ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,973 INFO server.Server: jetty-9.3.z-SNAPSHOT\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:38,985 INFO server.Server: Started @5950ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,004 INFO server.AbstractConnector: Started ServerConnector@234e121{HTTP/1.1,[http/1.1]}{0.0.0.0:4040}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,004 INFO util.Utils: Successfully started service 'SparkUI' on port 4040.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,030 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@44101dd7{/jobs,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,030 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@1a2e440c{/jobs/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,031 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@689003e3{/jobs/job,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,031 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6dfb6c2b{/jobs/job/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,032 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@24867721{/stages,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,032 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6ad3351a{/stages/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,033 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@1faedbf7{/stages/stage,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,034 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@198afe2b{/stages/stage/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,035 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@2e7dbda4{/stages/pool,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,035 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7a8f1f2{/stages/pool/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,036 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@60058f88{/storage,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,036 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@752f90{/storage/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,037 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5b37b5ac{/storage/rdd,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,037 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@215e7a02{/storage/rdd/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,038 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@402f6648{/environment,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,038 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6c1a4792{/environment/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,039 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@683fa282{/executors,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,040 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@5dcb81bc{/executors/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,040 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@57efe51b{/executors/threadDump,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,041 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@2ee70eb8{/executors/threadDump/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,048 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@4f33e334{/static,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,049 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7e090079{/,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,050 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@20c4a46b{/api,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,050 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@6821d75c{/jobs/job/kill,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,051 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@7d37d249{/stages/stage/kill,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,053 INFO ui.SparkUI: Bound SparkUI to 0.0.0.0, and started at http://10.0.246.193:4040\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:39,937 INFO client.RMProxy: Connecting to ResourceManager at /10.0.246.193:8032\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,141 INFO yarn.Client: Requesting a new application from cluster with 1 NodeManagers\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,197 INFO conf.Configuration: resource-types.xml not found\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,197 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,203 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (31706 MB per container)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,204 INFO yarn.Client: Will allocate AM container, with 896 MB memory including 384 MB overhead\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,204 INFO yarn.Client: Setting up container launch context for our AM\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,207 INFO yarn.Client: Setting up the launch environment for our AM container\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:40,212 INFO yarn.Client: Preparing resources for our AM container\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:41,064 WARN yarn.Client: Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m2020-06-11 18:51:41,723 INFO yarn.Client: Uploading resource file:/tmp/spark-e71b1312-0e3f-4add-97f0-41372025980a/__spark_libs__6025361934952542824.zip -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/__spark_libs__6025361934952542824.zip\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,205 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-spark_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.mleap_mleap-spark_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,234 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-spark-base_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.mleap_mleap-spark-base_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,255 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-runtime_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.mleap_mleap-runtime_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,285 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-core_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.mleap_mleap-core_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,706 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.bundle_bundle-ml_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.bundle_bundle-ml_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,726 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.scala-lang_scala-reflect-2.11.8.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.scala-lang_scala-reflect-2.11.8.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,768 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-base_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.mleap_mleap-base_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,789 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-tensor_2.11-0.8.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/ml.combust.mleap_mleap-tensor_2.11-0.8.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,807 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.apache.spark_spark-mllib-local_2.11-2.2.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.apache.spark_spark-mllib-local_2.11-2.2.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,825 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.github.rwl_jtransforms-2.4.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.github.rwl_jtransforms-2.4.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,843 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/io.spray_spray-json_2.11-1.3.2.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/io.spray_spray-json_2.11-1.3.2.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,859 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.scalanlp_breeze_2.11-0.13.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.scalanlp_breeze_2.11-0.13.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,937 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.apache.commons_commons-math3-3.4.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.apache.commons_commons-math3-3.4.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,967 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.apache.spark_spark-tags_2.11-2.2.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.apache.spark_spark-tags_2.11-2.2.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:43,985 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.spark-project.spark_unused-1.0.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.spark-project.spark_unused-1.0.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,005 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.scalanlp_breeze-macros_2.11-0.13.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.scalanlp_breeze-macros_2.11-0.13.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,028 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.github.fommil.netlib_core-1.1.2.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.github.fommil.netlib_core-1.1.2.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,044 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/net.sourceforge.f2j_arpack_combined_all-0.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/net.sourceforge.f2j_arpack_combined_all-0.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,074 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/net.sf.opencsv_opencsv-2.3.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/net.sf.opencsv_opencsv-2.3.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,091 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.spire-math_spire_2.11-0.13.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.spire-math_spire_2.11-0.13.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,144 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.chuusai_shapeless_2.11-2.3.2.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.chuusai_shapeless_2.11-2.3.2.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,183 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.slf4j_slf4j-api-1.7.16.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,221 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/junit_junit-4.12.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/junit_junit-4.12.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,236 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.hamcrest_hamcrest-core-1.3.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.hamcrest_hamcrest-core-1.3.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,347 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.spire-math_spire-macros_2.11-0.13.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.spire-math_spire-macros_2.11-0.13.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,765 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.typelevel_machinist_2.11-0.6.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.typelevel_machinist_2.11-0.6.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,781 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/org.typelevel_macro-compat_2.11-1.1.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/org.typelevel_macro-compat_2.11-1.1.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,796 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.trueaccord.scalapb_scalapb-runtime_2.11-0.6.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.trueaccord.scalapb_scalapb-runtime_2.11-0.6.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,826 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.jsuereth_scala-arm_2.11-2.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.jsuereth_scala-arm_2.11-2.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,841 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.typesafe_config-1.3.0.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.typesafe_config-1.3.0.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,861 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.trueaccord.lenses_lenses_2.11-0.4.12.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.trueaccord.lenses_lenses_2.11-0.4.12.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,876 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.lihaoyi_fastparse_2.11-0.4.2.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.lihaoyi_fastparse_2.11-0.4.2.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,893 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.google.protobuf_protobuf-java-3.3.1.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.google.protobuf_protobuf-java-3.3.1.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,910 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.lihaoyi_fastparse-utils_2.11-0.4.2.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.lihaoyi_fastparse-utils_2.11-0.4.2.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,924 INFO yarn.Client: Uploading resource file:/root/.ivy2/jars/com.lihaoyi_sourcecode_2.11-0.1.3.jar -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/com.lihaoyi_sourcecode_2.11-0.1.3.jar\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,940 INFO yarn.Client: Uploading resource file:/usr/spark-2.2.0/python/lib/pyspark.zip -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/pyspark.zip\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,956 INFO yarn.Client: Uploading resource file:/usr/spark-2.2.0/python/lib/py4j-0.10.4-src.zip -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/py4j-0.10.4-src.zip\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-spark_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-spark-base_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-runtime_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-core_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.bundle_bundle-ml_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.scala-lang_scala-reflect-2.11.8.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-base_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/ml.combust.mleap_mleap-tensor_2.11-0.8.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,972 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.apache.spark_spark-mllib-local_2.11-2.2.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.github.rwl_jtransforms-2.4.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/io.spray_spray-json_2.11-1.3.2.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.scalanlp_breeze_2.11-0.13.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.apache.commons_commons-math3-3.4.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.apache.spark_spark-tags_2.11-2.2.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.spark-project.spark_unused-1.0.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.scalanlp_breeze-macros_2.11-0.13.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.github.fommil.netlib_core-1.1.2.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/net.sourceforge.f2j_arpack_combined_all-0.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/net.sf.opencsv_opencsv-2.3.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.spire-math_spire_2.11-0.13.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.chuusai_shapeless_2.11-2.3.2.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.slf4j_slf4j-api-1.7.16.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/junit_junit-4.12.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.hamcrest_hamcrest-core-1.3.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.spire-math_spire-macros_2.11-0.13.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.typelevel_machinist_2.11-0.6.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/org.typelevel_macro-compat_2.11-1.1.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.trueaccord.scalapb_scalapb-runtime_2.11-0.6.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.jsuereth_scala-arm_2.11-2.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.typesafe_config-1.3.0.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.trueaccord.lenses_lenses_2.11-0.4.12.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.lihaoyi_fastparse_2.11-0.4.2.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.google.protobuf_protobuf-java-3.3.1.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.lihaoyi_fastparse-utils_2.11-0.4.2.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,973 WARN yarn.Client: Same path resource file:/root/.ivy2/jars/com.lihaoyi_sourcecode_2.11-0.1.3.jar added multiple times to distributed cache.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:44,988 INFO yarn.Client: Uploading resource file:/tmp/spark-e71b1312-0e3f-4add-97f0-41372025980a/__spark_conf__6813108728427663039.zip -> hdfs://10.0.246.193/user/root/.sparkStaging/application_1591901490552_0001/__spark_conf__.zip\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,018 INFO spark.SecurityManager: Changing view acls to: root\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,018 INFO spark.SecurityManager: Changing modify acls to: root\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,018 INFO spark.SecurityManager: Changing view acls groups to: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,018 INFO spark.SecurityManager: Changing modify acls groups to: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,018 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(root); groups with view permissions: Set(); users  with modify permissions: Set(root); groups with modify permissions: Set()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,022 INFO yarn.Client: Submitting application application_1591901490552_0001 to ResourceManager\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,200 INFO impl.YarnClientImpl: Submitted application application_1591901490552_0001\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:45,202 INFO cluster.SchedulerExtensionServices: Starting Yarn extension services with app application_1591901490552_0001 and attemptId None\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:46,207 INFO yarn.Client: Application report for application_1591901490552_0001 (state: ACCEPTED)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:46,209 INFO yarn.Client: \u001b[0m\n",
      "\u001b[34m#011 client token: N/A\u001b[0m\n",
      "\u001b[34m#011 diagnostics: [Thu Jun 11 18:51:45 +0000 2020] Scheduler has assigned a container for AM, waiting for AM container to be launched\u001b[0m\n",
      "\u001b[34m#011 ApplicationMaster host: N/A\u001b[0m\n",
      "\u001b[34m#011 ApplicationMaster RPC port: -1\u001b[0m\n",
      "\u001b[34m#011 queue: default\u001b[0m\n",
      "\u001b[34m#011 start time: 1591901505114\u001b[0m\n",
      "\u001b[34m#011 final status: UNDEFINED\u001b[0m\n",
      "\u001b[34m#011 tracking URL: http://algo-1:8088/proxy/application_1591901490552_0001/\u001b[0m\n",
      "\u001b[34m#011 user: root\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:47,211 INFO yarn.Client: Application report for application_1591901490552_0001 (state: ACCEPTED)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:48,213 INFO yarn.Client: Application report for application_1591901490552_0001 (state: ACCEPTED)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:49,216 INFO yarn.Client: Application report for application_1591901490552_0001 (state: ACCEPTED)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:50,219 INFO yarn.Client: Application report for application_1591901490552_0001 (state: ACCEPTED)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:51,193 INFO cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: ApplicationMaster registered as NettyRpcEndpointRef(spark-client://YarnAM)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:51,207 INFO cluster.YarnClientSchedulerBackend: Add WebUI Filter. org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter, Map(PROXY_HOSTS -> algo-1, PROXY_URI_BASES -> http://algo-1:8088/proxy/application_1591901490552_0001), /proxy/application_1591901490552_0001\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:51,208 INFO ui.JettyUtils: Adding filter: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:51,222 INFO yarn.Client: Application report for application_1591901490552_0001 (state: ACCEPTED)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m2020-06-11 18:51:52,224 INFO yarn.Client: Application report for application_1591901490552_0001 (state: RUNNING)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,224 INFO yarn.Client: \u001b[0m\n",
      "\u001b[34m#011 client token: N/A\u001b[0m\n",
      "\u001b[34m#011 diagnostics: N/A\u001b[0m\n",
      "\u001b[34m#011 ApplicationMaster host: 10.0.220.83\u001b[0m\n",
      "\u001b[34m#011 ApplicationMaster RPC port: 0\u001b[0m\n",
      "\u001b[34m#011 queue: default\u001b[0m\n",
      "\u001b[34m#011 start time: 1591901505114\u001b[0m\n",
      "\u001b[34m#011 final status: UNDEFINED\u001b[0m\n",
      "\u001b[34m#011 tracking URL: http://algo-1:8088/proxy/application_1591901490552_0001/\u001b[0m\n",
      "\u001b[34m#011 user: root\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,225 INFO cluster.YarnClientSchedulerBackend: Application application_1591901490552_0001 has started running.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,256 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45531.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,256 INFO netty.NettyBlockTransferService: Server created on 10.0.246.193:45531\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,257 INFO storage.BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,259 INFO storage.BlockManagerMaster: Registering BlockManager BlockManagerId(driver, 10.0.246.193, 45531, None)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,261 INFO storage.BlockManagerMasterEndpoint: Registering block manager 10.0.246.193:45531 with 366.3 MB RAM, BlockManagerId(driver, 10.0.246.193, 45531, None)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,264 INFO storage.BlockManagerMaster: Registered BlockManager BlockManagerId(driver, 10.0.246.193, 45531, None)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,264 INFO storage.BlockManager: Initialized BlockManager: BlockManagerId(driver, 10.0.246.193, 45531, None)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:52,275 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@14723068{/metrics/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:54,370 INFO cluster.YarnSchedulerBackend$YarnDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.0.220.83:47486) with ID 1\u001b[0m\n",
      "\u001b[34m2020-06-11 18:51:54,426 INFO storage.BlockManagerMasterEndpoint: Registering block manager algo-2:45985 with 11.9 GB RAM, BlockManagerId(1, algo-2, 45985, None)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,205 INFO cluster.YarnClientSchedulerBackend: SchedulerBackend is ready for scheduling beginning after waiting maxRegisteredResourcesWaitingTime: 30000(ms)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,371 INFO internal.SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/usr/spark-2.2.0/spark-warehouse').\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,372 INFO internal.SharedState: Warehouse path is 'file:/usr/spark-2.2.0/spark-warehouse'.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,376 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@13b8abbe{/SQL,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,376 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@642bbce6{/SQL/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,376 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@2ce3c916{/SQL/execution,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,377 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@28865b44{/SQL/execution/json,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,377 INFO handler.ContextHandler: Started o.s.j.s.ServletContextHandler@62758d45{/static/sql,null,AVAILABLE,@Spark}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:09,666 INFO state.StateStoreCoordinatorRef: Registered StateStoreCoordinator endpoint\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:10,562 INFO Configuration.deprecation: fs.s3a.server-side-encryption-key is deprecated. Instead, use fs.s3a.server-side-encryption.key\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,066 INFO datasources.FileSourceStrategy: Pruning directories with: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,068 INFO datasources.FileSourceStrategy: Post-Scan Filters: AtLeastNNulls(n, sex#0)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,070 INFO datasources.FileSourceStrategy: Output Data Schema: struct<sex: string>\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,075 INFO execution.FileSourceScanExec: Pushed Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,440 INFO codegen.CodeGenerator: Code generated in 143.388832 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,489 INFO memory.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 444.5 KB, free 365.9 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,529 INFO memory.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 41.5 KB, free 365.8 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,530 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on 10.0.246.193:45531 (size: 41.5 KB, free: 366.3 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,535 INFO spark.SparkContext: Created broadcast 0 from rdd at StringIndexer.scala:111\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,555 INFO execution.FileSourceScanExec: Planning scan with bin packing, max size: 4194304 bytes, open cost is considered as scanning 4194304 bytes.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,694 INFO spark.SparkContext: Starting job: countByValue at StringIndexer.scala:113\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,785 INFO scheduler.DAGScheduler: Registering RDD 6 (countByValue at StringIndexer.scala:113)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,787 INFO scheduler.DAGScheduler: Got job 0 (countByValue at StringIndexer.scala:113) with 1 output partitions\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,787 INFO scheduler.DAGScheduler: Final stage: ResultStage 1 (countByValue at StringIndexer.scala:113)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,787 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,788 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 0)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,792 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[6] at countByValue at StringIndexer.scala:113), which has no missing parents\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,859 INFO memory.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 16.5 KB, free 365.8 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,862 INFO memory.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 8.5 KB, free 365.8 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,862 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on 10.0.246.193:45531 (size: 8.5 KB, free: 366.3 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,864 INFO spark.SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1006\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,875 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[6] at countByValue at StringIndexer.scala:113) (first 15 tasks are for partitions Vector(0))\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,876 INFO cluster.YarnScheduler: Adding task set 0.0 with 1 tasks\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:11,896 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, algo-2, executor 1, partition 0, PROCESS_LOCAL, 5357 bytes)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:12,071 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on algo-2:45985 (size: 8.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:12,784 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on algo-2:45985 (size: 41.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,607 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 2719 ms on algo-2 (executor 1) (1/1)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,609 INFO cluster.YarnScheduler: Removed TaskSet 0.0, whose tasks have all completed, from pool \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,615 INFO scheduler.DAGScheduler: ShuffleMapStage 0 (countByValue at StringIndexer.scala:113) finished in 2.727 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,615 INFO scheduler.DAGScheduler: looking for newly runnable stages\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,615 INFO scheduler.DAGScheduler: running: Set()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,616 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 1)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,616 INFO scheduler.DAGScheduler: failed: Set()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,619 INFO scheduler.DAGScheduler: Submitting ResultStage 1 (ShuffledRDD[7] at countByValue at StringIndexer.scala:113), which has no missing parents\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,624 INFO memory.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 3.2 KB, free 365.8 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,626 INFO memory.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 1963.0 B, free 365.8 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,626 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on 10.0.246.193:45531 (size: 1963.0 B, free: 366.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,627 INFO spark.SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1006\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,628 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (ShuffledRDD[7] at countByValue at StringIndexer.scala:113) (first 15 tasks are for partitions Vector(0))\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,628 INFO cluster.YarnScheduler: Adding task set 1.0 with 1 tasks\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,633 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, algo-2, executor 1, partition 0, NODE_LOCAL, 4632 bytes)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,667 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on algo-2:45985 (size: 1963.0 B, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,687 INFO spark.MapOutputTrackerMasterEndpoint: Asked to send map output locations for shuffle 0 to 10.0.220.83:47486\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,689 INFO spark.MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 137 bytes\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,734 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 103 ms on algo-2 (executor 1) (1/1)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,734 INFO cluster.YarnScheduler: Removed TaskSet 1.0, whose tasks have all completed, from pool \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,735 INFO scheduler.DAGScheduler: ResultStage 1 (countByValue at StringIndexer.scala:113) finished in 0.104 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:14,738 INFO scheduler.DAGScheduler: Job 0 finished: countByValue at StringIndexer.scala:113, took 3.044059 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,154 INFO datasources.FileSourceStrategy: Pruning directories with: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,154 INFO datasources.FileSourceStrategy: Post-Scan Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,155 INFO datasources.FileSourceStrategy: Output Data Schema: struct<sex: string, length: double, diameter: double, height: double, whole_weight: double ... 7 more fields>\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,155 INFO execution.FileSourceScanExec: Pushed Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,309 INFO codegen.CodeGenerator: Code generated in 119.952667 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,327 INFO memory.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 444.5 KB, free 365.4 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,340 INFO memory.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 41.5 KB, free 365.3 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,340 INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on 10.0.246.193:45531 (size: 41.5 KB, free: 366.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,341 INFO spark.SparkContext: Created broadcast 3 from javaToPython at NativeMethodAccessorImpl.java:0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,341 INFO execution.FileSourceScanExec: Planning scan with bin packing, max size: 4194304 bytes, open cost is considered as scanning 4194304 bytes.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,491 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 2\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:15,491 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,071 INFO spark.SparkContext: Starting job: saveAsTextFile at NativeMethodAccessorImpl.java:0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,071 INFO scheduler.DAGScheduler: Got job 1 (saveAsTextFile at NativeMethodAccessorImpl.java:0) with 1 output partitions\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,072 INFO scheduler.DAGScheduler: Final stage: ResultStage 2 (saveAsTextFile at NativeMethodAccessorImpl.java:0)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,072 INFO scheduler.DAGScheduler: Parents of final stage: List()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,072 INFO scheduler.DAGScheduler: Missing parents: List()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,073 INFO scheduler.DAGScheduler: Submitting ResultStage 2 (MapPartitionsRDD[14] at saveAsTextFile at NativeMethodAccessorImpl.java:0), which has no missing parents\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,092 INFO memory.MemoryStore: Block broadcast_4 stored as values in memory (estimated size 174.2 KB, free 365.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,093 INFO memory.MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 64.1 KB, free 365.1 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,094 INFO storage.BlockManagerInfo: Added broadcast_4_piece0 in memory on 10.0.246.193:45531 (size: 64.1 KB, free: 366.1 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,094 INFO spark.SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1006\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,095 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 2 (MapPartitionsRDD[14] at saveAsTextFile at NativeMethodAccessorImpl.java:0) (first 15 tasks are for partitions Vector(0))\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,095 INFO cluster.YarnScheduler: Adding task set 2.0 with 1 tasks\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,095 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 2.0 (TID 2, algo-2, executor 1, partition 0, PROCESS_LOCAL, 5368 bytes)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,106 INFO storage.BlockManagerInfo: Added broadcast_4_piece0 in memory on algo-2:45985 (size: 64.1 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:16,928 INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on algo-2:45985 (size: 41.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:19,933 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 2.0 (TID 2) in 3838 ms on algo-2 (executor 1) (1/1)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:19,934 INFO cluster.YarnScheduler: Removed TaskSet 2.0, whose tasks have all completed, from pool \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:19,935 INFO scheduler.DAGScheduler: ResultStage 2 (saveAsTextFile at NativeMethodAccessorImpl.java:0) finished in 3.840 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:19,936 INFO scheduler.DAGScheduler: Job 1 finished: saveAsTextFile at NativeMethodAccessorImpl.java:0, took 3.865047 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,462 INFO spark.ContextCleaner: Cleaned accumulator 4\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,462 INFO spark.ContextCleaner: Cleaned accumulator 1\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,498 INFO spark.ContextCleaner: Cleaned shuffle 0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,507 INFO storage.BlockManagerInfo: Removed broadcast_4_piece0 on 10.0.246.193:45531 in memory (size: 64.1 KB, free: 366.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,512 INFO storage.BlockManagerInfo: Removed broadcast_4_piece0 on algo-2:45985 in memory (size: 64.1 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,517 INFO storage.BlockManagerInfo: Removed broadcast_2_piece0 on 10.0.246.193:45531 in memory (size: 1963.0 B, free: 366.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,520 INFO storage.BlockManagerInfo: Removed broadcast_2_piece0 on algo-2:45985 in memory (size: 1963.0 B, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,522 INFO spark.ContextCleaner: Cleaned accumulator 3\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,522 INFO spark.ContextCleaner: Cleaned accumulator 5\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,522 INFO spark.ContextCleaner: Cleaned accumulator 6\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,523 INFO storage.BlockManagerInfo: Removed broadcast_0_piece0 on 10.0.246.193:45531 in memory (size: 41.5 KB, free: 366.3 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,524 INFO storage.BlockManagerInfo: Removed broadcast_0_piece0 on algo-2:45985 in memory (size: 41.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,528 INFO storage.BlockManagerInfo: Removed broadcast_1_piece0 on 10.0.246.193:45531 in memory (size: 8.5 KB, free: 366.3 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,529 INFO storage.BlockManagerInfo: Removed broadcast_1_piece0 on algo-2:45985 in memory (size: 8.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,532 INFO spark.ContextCleaner: Cleaned accumulator 2\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,686 INFO datasources.FileSourceStrategy: Pruning directories with: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,686 INFO datasources.FileSourceStrategy: Post-Scan Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,687 INFO datasources.FileSourceStrategy: Output Data Schema: struct<sex: string, length: double, diameter: double, height: double, whole_weight: double ... 7 more fields>\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,687 INFO execution.FileSourceScanExec: Pushed Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,781 INFO codegen.CodeGenerator: Code generated in 76.629188 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,799 INFO memory.MemoryStore: Block broadcast_5 stored as values in memory (estimated size 444.5 KB, free 365.4 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,815 INFO memory.MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 41.5 KB, free 365.4 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,815 INFO storage.BlockManagerInfo: Added broadcast_5_piece0 in memory on 10.0.246.193:45531 (size: 41.5 KB, free: 366.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,816 INFO spark.SparkContext: Created broadcast 5 from javaToPython at NativeMethodAccessorImpl.java:0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:20,816 INFO execution.FileSourceScanExec: Planning scan with bin packing, max size: 4194304 bytes, open cost is considered as scanning 4194304 bytes.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,060 INFO output.FileOutputCommitter: File Output Committer Algorithm version is 2\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,060 INFO output.FileOutputCommitter: FileOutputCommitter skip cleanup _temporary folders under output directory:false, ignore cleanup failures: false\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,543 INFO spark.SparkContext: Starting job: saveAsTextFile at NativeMethodAccessorImpl.java:0\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,543 INFO scheduler.DAGScheduler: Got job 2 (saveAsTextFile at NativeMethodAccessorImpl.java:0) with 1 output partitions\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,543 INFO scheduler.DAGScheduler: Final stage: ResultStage 3 (saveAsTextFile at NativeMethodAccessorImpl.java:0)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,543 INFO scheduler.DAGScheduler: Parents of final stage: List()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,544 INFO scheduler.DAGScheduler: Missing parents: List()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,544 INFO scheduler.DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[21] at saveAsTextFile at NativeMethodAccessorImpl.java:0), which has no missing parents\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,560 INFO memory.MemoryStore: Block broadcast_6 stored as values in memory (estimated size 174.2 KB, free 365.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,562 INFO memory.MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 64.1 KB, free 365.1 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,563 INFO storage.BlockManagerInfo: Added broadcast_6_piece0 in memory on 10.0.246.193:45531 (size: 64.1 KB, free: 366.2 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,563 INFO spark.SparkContext: Created broadcast 6 from broadcast at DAGScheduler.scala:1006\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,564 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[21] at saveAsTextFile at NativeMethodAccessorImpl.java:0) (first 15 tasks are for partitions Vector(0))\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,564 INFO cluster.YarnScheduler: Adding task set 3.0 with 1 tasks\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,565 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 3.0 (TID 3, algo-2, executor 1, partition 0, PROCESS_LOCAL, 5368 bytes)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,619 INFO storage.BlockManagerInfo: Added broadcast_6_piece0 in memory on algo-2:45985 (size: 64.1 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:21,818 INFO storage.BlockManagerInfo: Added broadcast_5_piece0 in memory on algo-2:45985 (size: 41.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:24,040 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 3.0 (TID 3) in 2476 ms on algo-2 (executor 1) (1/1)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:24,040 INFO cluster.YarnScheduler: Removed TaskSet 3.0, whose tasks have all completed, from pool \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:24,041 INFO scheduler.DAGScheduler: ResultStage 3 (saveAsTextFile at NativeMethodAccessorImpl.java:0) finished in 2.477 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:24,042 INFO scheduler.DAGScheduler: Job 2 finished: saveAsTextFile at NativeMethodAccessorImpl.java:0, took 2.499029 s\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m2020-06-11 18:52:25,042 INFO datasources.FileSourceStrategy: Pruning directories with: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,043 INFO datasources.FileSourceStrategy: Post-Scan Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,043 INFO datasources.FileSourceStrategy: Output Data Schema: struct<sex: string, length: double, diameter: double, height: double, whole_weight: double ... 7 more fields>\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,043 INFO execution.FileSourceScanExec: Pushed Filters: \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,135 INFO codegen.CodeGenerator: Code generated in 54.634582 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,152 INFO memory.MemoryStore: Block broadcast_7 stored as values in memory (estimated size 444.5 KB, free 364.7 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,167 INFO memory.MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 41.5 KB, free 364.6 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,168 INFO storage.BlockManagerInfo: Added broadcast_7_piece0 in memory on 10.0.246.193:45531 (size: 41.5 KB, free: 366.1 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,169 INFO spark.SparkContext: Created broadcast 7 from sparkToMleapDataShape at VectorAssemblerOp.scala:26\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,170 INFO execution.FileSourceScanExec: Planning scan with bin packing, max size: 4194304 bytes, open cost is considered as scanning 4194304 bytes.\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,180 INFO spark.SparkContext: Starting job: sparkToMleapDataShape at VectorAssemblerOp.scala:26\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,180 INFO scheduler.DAGScheduler: Got job 3 (sparkToMleapDataShape at VectorAssemblerOp.scala:26) with 1 output partitions\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,180 INFO scheduler.DAGScheduler: Final stage: ResultStage 4 (sparkToMleapDataShape at VectorAssemblerOp.scala:26)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,180 INFO scheduler.DAGScheduler: Parents of final stage: List()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,181 INFO scheduler.DAGScheduler: Missing parents: List()\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,181 INFO scheduler.DAGScheduler: Submitting ResultStage 4 (MapPartitionsRDD[24] at sparkToMleapDataShape at VectorAssemblerOp.scala:26), which has no missing parents\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,184 INFO memory.MemoryStore: Block broadcast_8 stored as values in memory (estimated size 56.4 KB, free 364.6 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,186 INFO memory.MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 20.7 KB, free 364.6 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,186 INFO storage.BlockManagerInfo: Added broadcast_8_piece0 in memory on 10.0.246.193:45531 (size: 20.7 KB, free: 366.1 MB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,186 INFO spark.SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:1006\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,187 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[24] at sparkToMleapDataShape at VectorAssemblerOp.scala:26) (first 15 tasks are for partitions Vector(0))\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,187 INFO cluster.YarnScheduler: Adding task set 4.0 with 1 tasks\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,188 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 4.0 (TID 4, algo-2, executor 1, partition 0, PROCESS_LOCAL, 5368 bytes)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,197 INFO storage.BlockManagerInfo: Added broadcast_8_piece0 in memory on algo-2:45985 (size: 20.7 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,262 INFO storage.BlockManagerInfo: Added broadcast_7_piece0 in memory on algo-2:45985 (size: 41.5 KB, free: 11.9 GB)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,399 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 4.0 (TID 4) in 211 ms on algo-2 (executor 1) (1/1)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,399 INFO cluster.YarnScheduler: Removed TaskSet 4.0, whose tasks have all completed, from pool \u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,400 INFO scheduler.DAGScheduler: ResultStage 4 (sparkToMleapDataShape at VectorAssemblerOp.scala:26) finished in 0.213 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,400 INFO scheduler.DAGScheduler: Job 3 finished: sparkToMleapDataShape at VectorAssemblerOp.scala:26, took 0.220304 s\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,420 INFO codegen.CodeGenerator: Code generated in 9.318428 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,680 INFO spark.SparkContext: Invoking stop() from shutdown hook\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,684 INFO server.AbstractConnector: Stopped Spark@234e121{HTTP/1.1,[http/1.1]}{0.0.0.0:4040}\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,686 INFO ui.SparkUI: Stopped Spark web UI at http://10.0.246.193:4040\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,690 INFO cluster.YarnClientSchedulerBackend: Interrupting monitor thread\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,701 INFO cluster.YarnClientSchedulerBackend: Shutting down all executors\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,701 INFO cluster.YarnSchedulerBackend$YarnDriverEndpoint: Asking each executor to shut down\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,704 INFO cluster.SchedulerExtensionServices: Stopping SchedulerExtensionServices\u001b[0m\n",
      "\u001b[34m(serviceOption=None,\n",
      " services=List(),\n",
      " started=false)\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,706 INFO cluster.YarnClientSchedulerBackend: Stopped\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,709 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,721 INFO memory.MemoryStore: MemoryStore cleared\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,722 INFO storage.BlockManager: BlockManager stopped\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,723 INFO storage.BlockManagerMaster: BlockManagerMaster stopped\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,725 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,750 INFO spark.SparkContext: Successfully stopped SparkContext\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,751 INFO util.ShutdownHookManager: Shutdown hook called\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,751 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-e71b1312-0e3f-4add-97f0-41372025980a/pyspark-41e8c821-82b4-45bc-b32b-c36466284f98\u001b[0m\n",
      "\u001b[34m2020-06-11 18:52:25,751 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-e71b1312-0e3f-4add-97f0-41372025980a\u001b[0m\n",
      "\u001b[35m2020-06-11 18:52:27\u001b[0m\n",
      "\u001b[35mFinished Yarn configuration files setup.\n",
      "\u001b[0m\n",
      "\u001b[35mReceived end of job signal, exiting...\u001b[0m\n",
      "\u001b[34mFinished Yarn configuration files setup.\n",
      "\u001b[0m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.processing import ScriptProcessor, ProcessingInput\n",
    "spark_processor = ScriptProcessor(base_job_name='spark-preprocessor',\n",
    "                                  image_uri=spark_repository_uri,\n",
    "                                  command=['/opt/program/submit'],\n",
    "                                  role=role,\n",
    "                                  instance_count=2,\n",
    "                                  instance_type='ml.r5.xlarge',\n",
    "                                  max_runtime_in_seconds=1200,\n",
    "                                  env={'mode': 'python'})\n",
    "\n",
    "spark_processor.run(code='preprocess.py',\n",
    "                    arguments=['s3_input_bucket', bucket,\n",
    "                               's3_input_key_prefix', input_prefix,\n",
    "                               's3_output_bucket', bucket,\n",
    "                               's3_output_key_prefix', input_preprocessed_prefix,\n",
    "                               's3_model_bucket', bucket,\n",
    "                               's3_mleap_model_prefix', mleap_model_prefix],\n",
    "                    logs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 5 rows from s3://sagemaker-us-east-1-115731758279/sagemaker/spark-preprocess-demo/2020-06-11-18-47-41/input/preprocessed/abalone/train/\n",
      "6.0,0.0,0.0,0.29,0.21,0.075,0.275,0.113,0.0675,0.035\n",
      "5.0,0.0,0.0,0.29,0.225,0.075,0.14,0.0515,0.0235,0.04\n",
      "7.0,0.0,0.0,0.305,0.225,0.07,0.1485,0.0585,0.0335,0.045\n",
      "7.0,0.0,0.0,0.305,0.23,0.08,0.156,0.0675,0.0345,0.048\n",
      "7.0,0.0,0.0,0.325,0.26,0.09,0.1915,0.085,0.036,0.062\n"
     ]
    }
   ],
   "source": [
    "print('Top 5 rows from s3://{}/{}/train/'.format(bucket, input_preprocessed_prefix))\n",
    "!aws s3 cp --quiet s3://$bucket/$input_preprocessed_prefix/train/part-00000 - | head -n5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "683313688378.dkr.ecr.us-east-1.amazonaws.com/sagemaker-xgboost:0.90-1-cpu-py3\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.amazon.amazon_estimator import get_image_uri\n",
    "\n",
    "training_image = get_image_uri(sagemaker_session.boto_region_name, 'xgboost', repo_version=\"0.90-1\")\n",
    "print(training_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_train_data = 's3://{}/{}/{}'.format(bucket, input_preprocessed_prefix, 'train/part')\n",
    "s3_validation_data = 's3://{}/{}/{}'.format(bucket, input_preprocessed_prefix, 'validation/part')\n",
    "s3_output_location = 's3://{}/{}/{}'.format(bucket, prefix, 'xgboost_model')\n",
    "\n",
    "xgb_model = sagemaker.estimator.Estimator(training_image,\n",
    "                                          role, \n",
    "                                          train_instance_count=1, \n",
    "                                          train_instance_type='ml.m4.xlarge',\n",
    "                                          train_volume_size = 20,\n",
    "                                          train_max_run = 3600,\n",
    "                                          input_mode= 'File',\n",
    "                                          output_path=s3_output_location,\n",
    "                                          sagemaker_session=sagemaker_session)\n",
    "\n",
    "xgb_model.set_hyperparameters(objective = \"reg:linear\",\n",
    "                              eta = .2,\n",
    "                              gamma = 4,\n",
    "                              max_depth = 5,\n",
    "                              num_round = 10,\n",
    "                              subsample = 0.7,\n",
    "                              silent = 0,\n",
    "                              min_child_weight = 6)\n",
    "\n",
    "train_data = sagemaker.session.s3_input(s3_train_data, distribution='FullyReplicated', \n",
    "                        content_type='text/csv', s3_data_type='S3Prefix')\n",
    "validation_data = sagemaker.session.s3_input(s3_validation_data, distribution='FullyReplicated', \n",
    "                             content_type='text/csv', s3_data_type='S3Prefix')\n",
    "\n",
    "data_channels = {'train': train_data, 'validation': validation_data}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-06-11 18:54:02 Starting - Starting the training job...\n",
      "2020-06-11 18:54:04 Starting - Launching requested ML instances.........\n",
      "2020-06-11 18:55:51 Starting - Preparing the instances for training......\n",
      "2020-06-11 18:56:45 Downloading - Downloading input data...\n",
      "2020-06-11 18:57:09 Training - Downloading the training image..\u001b[34mINFO:sagemaker-containers:Imported framework sagemaker_xgboost_container.training\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-containers:Failed to parse hyperparameter objective value reg:linear to Json.\u001b[0m\n",
      "\u001b[34mReturning the value itself\u001b[0m\n",
      "\u001b[34mINFO:sagemaker-containers:No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[34mINFO:sagemaker_xgboost_container.training:Running XGBoost Sagemaker in algorithm mode\u001b[0m\n",
      "\u001b[34mINFO:root:Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[34mINFO:root:Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[34mINFO:root:Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[34m[18:57:36] 3347x9 matrix with 30123 entries loaded from /opt/ml/input/data/train?format=csv&label_column=0&delimiter=,\u001b[0m\n",
      "\u001b[34mINFO:root:Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[34m[18:57:36] 830x9 matrix with 7470 entries loaded from /opt/ml/input/data/validation?format=csv&label_column=0&delimiter=,\u001b[0m\n",
      "\u001b[34mINFO:root:Single node training.\u001b[0m\n",
      "\u001b[34mINFO:root:Train matrix has 3347 rows\u001b[0m\n",
      "\u001b[34mINFO:root:Validation matrix has 830 rows\u001b[0m\n",
      "\u001b[34m[18:57:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\u001b[0m\n",
      "\u001b[34m[0]#011train-rmse:8.0507#011validation-rmse:8.29007\u001b[0m\n",
      "\u001b[34m[1]#011train-rmse:6.58603#011validation-rmse:6.82405\u001b[0m\n",
      "\u001b[34m[2]#011train-rmse:5.43068#011validation-rmse:5.66042\u001b[0m\n",
      "\u001b[34m[3]#011train-rmse:4.53301#011validation-rmse:4.78407\u001b[0m\n",
      "\u001b[34m[4]#011train-rmse:3.82399#011validation-rmse:4.1088\u001b[0m\n",
      "\u001b[34m[5]#011train-rmse:3.29428#011validation-rmse:3.59554\u001b[0m\n",
      "\u001b[34m[6]#011train-rmse:2.89225#011validation-rmse:3.21708\u001b[0m\n",
      "\u001b[34m[7]#011train-rmse:2.59668#011validation-rmse:2.95967\u001b[0m\n",
      "\u001b[34m[8]#011train-rmse:2.38373#011validation-rmse:2.76801\u001b[0m\n",
      "\u001b[34m[9]#011train-rmse:2.2359#011validation-rmse:2.64289\u001b[0m\n",
      "\n",
      "2020-06-11 18:57:46 Uploading - Uploading generated training model\n",
      "2020-06-11 18:57:46 Completed - Training job completed\n",
      "Training seconds: 61\n",
      "Billable seconds: 61\n"
     ]
    }
   ],
   "source": [
    "xgb_model.fit(inputs=data_channels, logs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"input\": [{\"name\": \"sex\", \"type\": \"string\"}, {\"name\": \"length\", \"type\": \"double\"}, {\"name\": \"diameter\", \"type\": \"double\"}, {\"name\": \"height\", \"type\": \"double\"}, {\"name\": \"whole_weight\", \"type\": \"double\"}, {\"name\": \"shucked_weight\", \"type\": \"double\"}, {\"name\": \"viscera_weight\", \"type\": \"double\"}, {\"name\": \"shell_weight\", \"type\": \"double\"}], \"output\": {\"name\": \"features\", \"type\": \"double\", \"struct\": \"vector\"}}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "schema = {\n",
    "    \"input\": [\n",
    "        {\n",
    "            \"name\": \"sex\",\n",
    "            \"type\": \"string\"\n",
    "        }, \n",
    "        {\n",
    "            \"name\": \"length\",\n",
    "            \"type\": \"double\"\n",
    "        }, \n",
    "        {\n",
    "            \"name\": \"diameter\",\n",
    "            \"type\": \"double\"\n",
    "        }, \n",
    "        {\n",
    "            \"name\": \"height\",\n",
    "            \"type\": \"double\"\n",
    "        }, \n",
    "        {\n",
    "            \"name\": \"whole_weight\",\n",
    "            \"type\": \"double\"\n",
    "        }, \n",
    "        {\n",
    "            \"name\": \"shucked_weight\",\n",
    "            \"type\": \"double\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"viscera_weight\",\n",
    "            \"type\": \"double\"\n",
    "        }, \n",
    "        {\n",
    "            \"name\": \"shell_weight\",\n",
    "            \"type\": \"double\"\n",
    "        }, \n",
    "    ],\n",
    "    \"output\": \n",
    "        {\n",
    "            \"name\": \"features\",\n",
    "            \"type\": \"double\",\n",
    "            \"struct\": \"vector\"\n",
    "        }\n",
    "}\n",
    "schema_json = json.dumps(schema)\n",
    "print(schema_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.model import Model\n",
    "from sagemaker.pipeline import PipelineModel\n",
    "from sagemaker.sparkml.model import SparkMLModel\n",
    "\n",
    "sparkml_data = 's3://{}/{}/{}'.format(bucket, mleap_model_prefix, 'model.tar.gz')\n",
    "# passing the schema defined above by using an environment variable that sagemaker-sparkml-serving understands\n",
    "sparkml_model = SparkMLModel(model_data=sparkml_data, env={'SAGEMAKER_SPARKML_SCHEMA' : schema_json})\n",
    "xgb_model = Model(model_data=xgb_model.model_data, image=training_image)\n",
    "\n",
    "model_name = 'inference-pipeline-' + timestamp_prefix\n",
    "sm_model = PipelineModel(name=model_name, role=role, models=[sparkml_model, xgb_model])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------!"
     ]
    }
   ],
   "source": [
    "endpoint_name = 'inference-pipeline-ep-' + timestamp_prefix\n",
    "sm_model.deploy(initial_instance_count=1, instance_type='ml.c4.xlarge', endpoint_name=endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'10.723018646240234'\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.predictor import json_serializer, csv_serializer, json_deserializer, RealTimePredictor\n",
    "from sagemaker.content_types import CONTENT_TYPE_CSV, CONTENT_TYPE_JSON\n",
    "payload = \"F,0.515,0.425,0.14,0.766,0.304,0.1725,0.255\"\n",
    "predictor = RealTimePredictor(endpoint=endpoint_name, sagemaker_session=sagemaker_session, serializer=csv_serializer,\n",
    "                                content_type=CONTENT_TYPE_CSV, accept=CONTENT_TYPE_CSV)\n",
    "print(predictor.predict(payload))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'10.723018646240234'\n"
     ]
    }
   ],
   "source": [
    "payload = {\"data\": [\"F\",0.515,0.425,0.14,0.766,0.304,0.1725,0.255]}\n",
    "predictor = RealTimePredictor(endpoint=endpoint_name, sagemaker_session=sagemaker_session, serializer=json_serializer,\n",
    "                                content_type=CONTENT_TYPE_JSON, accept=CONTENT_TYPE_CSV)\n",
    "\n",
    "print(predictor.predict(payload))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BATCH TRANSFORM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-06-11 19:12:47--  https://s3-us-west-2.amazonaws.com/sparkml-mleap/data/batch_input_abalone.csv\n",
      "Resolving s3-us-west-2.amazonaws.com (s3-us-west-2.amazonaws.com)... 52.218.233.136\n",
      "Connecting to s3-us-west-2.amazonaws.com (s3-us-west-2.amazonaws.com)|52.218.233.136|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 654 [text/csv]\n",
      "Saving to: ‘batch_input_abalone.csv’\n",
      "\n",
      "batch_input_abalone 100%[===================>]     654  --.-KB/s    in 0s      \n",
      "\n",
      "2020-06-11 19:12:47 (18.5 MB/s) - ‘batch_input_abalone.csv’ saved [654/654]\n",
      "\n",
      "\n",
      "\n",
      "Showing first five lines\n",
      "\n",
      "M,0.455,0.365,0.095,0.514,0.2245,0.101,0.15\n",
      "M,0.35,0.265,0.09,0.2255,0.0995,0.0485,0.07\n",
      "F,0.53,0.42,0.135,0.677,0.2565,0.1415,0.21\n",
      "M,0.44,0.365,0.125,0.516,0.2155,0.114,0.155\n",
      "I,0.33,0.255,0.08,0.205,0.0895,0.0395,0.055\n",
      "\n",
      "\n",
      "As we can see, it is identical to the training file apart from the label being absent here.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://s3-us-west-2.amazonaws.com/sparkml-mleap/data/batch_input_abalone.csv\n",
    "!printf \"\\n\\nShowing first five lines\\n\\n\"    \n",
    "!head -n 5 batch_input_abalone.csv \n",
    "!printf \"\\n\\nAs we can see, it is identical to the training file apart from the label being absent here.\\n\\n\"  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_input_loc = sagemaker_session.upload_data(path='batch_input_abalone.csv', bucket=bucket, key_prefix='batch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".........................\u001b[34m  .   ____          _            __ _ _\n",
      " /\\\\ / ___'_ __ _ _(_)_ __  __ _ \\ \\ \\ \\\u001b[0m\n",
      "\u001b[35m  .   ____          _            __ _ _\n",
      " /\\\\ / ___'_ __ _ _(_)_ __  __ _ \\ \\ \\ \\\u001b[0m\n",
      "\u001b[34m( ( )\\___ | '_ | '_| | '_ \\/ _` | \\ \\ \\ \\\n",
      " \\\\/  ___)| |_)| | | | | || (_| |  ) ) ) )\n",
      "  '  |____| .__|_| |_|_| |_\\__, | / / / /\n",
      " =========|_|==============|___/=/_/_/_/\n",
      " :: Spring Boot ::                  (v2.2)\n",
      "\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:47.519  INFO 7 --- [           main] com.amazonaws.sagemaker.App              : Starting App v2.2 on 37eea86bd301 with PID 7 (/usr/local/lib/sparkml-serving-2.2.jar started by root in /sagemaker-sparkml-model-server)\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:47.530  INFO 7 --- [           main] com.amazonaws.sagemaker.App              : No active profile set, falling back to default profiles: default\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.321  INFO 7 --- [           main] org.eclipse.jetty.util.log               : Logging initialized @3825ms to org.eclipse.jetty.util.log.Slf4jLog\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.586  INFO 7 --- [           main] o.s.b.w.e.j.JettyServletWebServerFactory : Server initialized with port: 8080\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.615  INFO 7 --- [           main] org.eclipse.jetty.server.Server          : jetty-9.4.z-SNAPSHOT; built: 2018-08-30T13:59:14.071Z; git: 27208684755d94a92186989f695db2d7b21ebc51; jvm 1.8.0_181-8u181-b13-2~deb9u1-b13\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.944  INFO 7 --- [           main] org.eclipse.jetty.server.session         : DefaultSessionIdManager workerName=node0\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.945  INFO 7 --- [           main] org.eclipse.jetty.server.session         : No SessionScavenger set, using defaults\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.950  INFO 7 --- [           main] org.eclipse.jetty.server.session         : node0 Scavenging every 660000ms\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.960  INFO 7 --- [           main] o.e.j.s.h.ContextHandler.application     : Initializing Spring embedded WebApplicationContext\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:50.966  INFO 7 --- [           main] o.s.web.context.ContextLoader            : Root WebApplicationContext: initialization completed in 3271 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.043  INFO 7 --- [           main] o.s.b.w.servlet.ServletRegistrationBean  : Servlet dispatcherServlet mapped to [/]\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.047  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'characterEncodingFilter' to: [/*]\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.049  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'hiddenHttpMethodFilter' to: [/*]\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.050  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'formContentFilter' to: [/*]\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.050  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'requestContextFilter' to: [/*]\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.064  INFO 7 --- [           main] o.e.jetty.server.handler.ContextHandler  : Started o.s.b.w.e.j.JettyEmbeddedWebAppContext@655a5d9c{application,/,[file:///tmp/jetty-docbase.3901132992416114828.8080/],AVAILABLE}\u001b[0m\n",
      "\u001b[35m( ( )\\___ | '_ | '_| | '_ \\/ _` | \\ \\ \\ \\\n",
      " \\\\/  ___)| |_)| | | | | || (_| |  ) ) ) )\n",
      "  '  |____| .__|_| |_|_| |_\\__, | / / / /\n",
      " =========|_|==============|___/=/_/_/_/\n",
      " :: Spring Boot ::                  (v2.2)\n",
      "\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:47.519  INFO 7 --- [           main] com.amazonaws.sagemaker.App              : Starting App v2.2 on 37eea86bd301 with PID 7 (/usr/local/lib/sparkml-serving-2.2.jar started by root in /sagemaker-sparkml-model-server)\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:47.530  INFO 7 --- [           main] com.amazonaws.sagemaker.App              : No active profile set, falling back to default profiles: default\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.321  INFO 7 --- [           main] org.eclipse.jetty.util.log               : Logging initialized @3825ms to org.eclipse.jetty.util.log.Slf4jLog\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.586  INFO 7 --- [           main] o.s.b.w.e.j.JettyServletWebServerFactory : Server initialized with port: 8080\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.615  INFO 7 --- [           main] org.eclipse.jetty.server.Server          : jetty-9.4.z-SNAPSHOT; built: 2018-08-30T13:59:14.071Z; git: 27208684755d94a92186989f695db2d7b21ebc51; jvm 1.8.0_181-8u181-b13-2~deb9u1-b13\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.944  INFO 7 --- [           main] org.eclipse.jetty.server.session         : DefaultSessionIdManager workerName=node0\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.945  INFO 7 --- [           main] org.eclipse.jetty.server.session         : No SessionScavenger set, using defaults\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.950  INFO 7 --- [           main] org.eclipse.jetty.server.session         : node0 Scavenging every 660000ms\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.960  INFO 7 --- [           main] o.e.j.s.h.ContextHandler.application     : Initializing Spring embedded WebApplicationContext\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:50.966  INFO 7 --- [           main] o.s.web.context.ContextLoader            : Root WebApplicationContext: initialization completed in 3271 ms\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.043  INFO 7 --- [           main] o.s.b.w.servlet.ServletRegistrationBean  : Servlet dispatcherServlet mapped to [/]\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.047  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'characterEncodingFilter' to: [/*]\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.049  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'hiddenHttpMethodFilter' to: [/*]\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.050  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'formContentFilter' to: [/*]\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.050  INFO 7 --- [           main] o.s.b.w.servlet.FilterRegistrationBean   : Mapping filter: 'requestContextFilter' to: [/*]\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.064  INFO 7 --- [           main] o.e.jetty.server.handler.ContextHandler  : Started o.s.b.w.e.j.JettyEmbeddedWebAppContext@655a5d9c{application,/,[file:///tmp/jetty-docbase.3901132992416114828.8080/],AVAILABLE}\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:51.065  INFO 7 --- [           main] org.eclipse.jetty.server.Server          : Started @4573ms\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:51.065  INFO 7 --- [           main] org.eclipse.jetty.server.Server          : Started @4573ms\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [14] [INFO] Starting gunicorn 19.10.0\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [14] [INFO] Listening at: unix:/tmp/gunicorn.sock (14)\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [14] [INFO] Using worker: gevent\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [21] [INFO] Booting worker with pid: 21\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [22] [INFO] Booting worker with pid: 22\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [23] [INFO] Booting worker with pid: 23\u001b[0m\n",
      "\u001b[32m[2020-06-11 19:16:47 +0000] [24] [INFO] Booting worker with pid: 24\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:53.768  INFO 7 --- [           main] o.s.s.concurrent.ThreadPoolTaskExecutor  : Initializing ExecutorService 'applicationTaskExecutor'\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:53.768  INFO 7 --- [           main] o.s.s.concurrent.ThreadPoolTaskExecutor  : Initializing ExecutorService 'applicationTaskExecutor'\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:54.095  INFO 7 --- [           main] o.e.j.s.h.ContextHandler.application     : Initializing Spring DispatcherServlet 'dispatcherServlet'\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:54.095  INFO 7 --- [           main] o.s.web.servlet.DispatcherServlet        : Initializing Servlet 'dispatcherServlet'\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:54.102  INFO 7 --- [           main] o.s.web.servlet.DispatcherServlet        : Completed initialization in 7 ms\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:54.131  INFO 7 --- [           main] o.e.jetty.server.AbstractConnector       : Started ServerConnector@261de205{HTTP/1.1,[http/1.1]}{0.0.0.0:8080}\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:54.138  INFO 7 --- [           main] o.s.b.web.embedded.jetty.JettyWebServer  : Jetty started on port(s) 8080 (http/1.1) with context path '/'\u001b[0m\n",
      "\u001b[34m2020-06-11 19:16:54.142  INFO 7 --- [           main] com.amazonaws.sagemaker.App              : Started App in 7.18 seconds (JVM running for 7.649)\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:54.095  INFO 7 --- [           main] o.e.j.s.h.ContextHandler.application     : Initializing Spring DispatcherServlet 'dispatcherServlet'\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:54.095  INFO 7 --- [           main] o.s.web.servlet.DispatcherServlet        : Initializing Servlet 'dispatcherServlet'\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:54.102  INFO 7 --- [           main] o.s.web.servlet.DispatcherServlet        : Completed initialization in 7 ms\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:54.131  INFO 7 --- [           main] o.e.jetty.server.AbstractConnector       : Started ServerConnector@261de205{HTTP/1.1,[http/1.1]}{0.0.0.0:8080}\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:54.138  INFO 7 --- [           main] o.s.b.web.embedded.jetty.JettyWebServer  : Jetty started on port(s) 8080 (http/1.1) with context path '/'\u001b[0m\n",
      "\u001b[35m2020-06-11 19:16:54.142  INFO 7 --- [           main] com.amazonaws.sagemaker.App              : Started App in 7.18 seconds (JVM running for 7.649)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[2020-06-11:19:16:56:INFO] No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:56 +0000] \"GET /ping HTTP/1.1\" 200 0 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:56 +0000] \"GET /execution-parameters HTTP/1.1\" 200 84 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[36m2020-06-11T19:16:56.670:[sagemaker logs]: MaxConcurrentTransforms=4, MaxPayloadInMB=5, BatchStrategy=SINGLE_RECORD\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m[19:16:56] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:56 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[19:16:56] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:56 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m[19:16:56] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:56 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:56:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:56 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m[19:16:57] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 19 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 17 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 19 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 19 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 18 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 19 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 19 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\u001b[32m[2020-06-11:19:16:57:INFO] Determined delimiter of CSV input is ','\u001b[0m\n",
      "\u001b[32m169.254.255.130 - - [11/Jun/2020:19:16:57 +0000] \"POST /invocations HTTP/1.1\" 200 17 \"-\" \"Go-http-client/1.1\"\u001b[0m\n",
      "\n"
     ]
    }
   ],
   "source": [
    "input_data_path = 's3://{}/{}/{}'.format(bucket, 'batch', 'batch_input_abalone.csv')\n",
    "output_data_path = 's3://{}/{}/{}'.format(bucket, 'batch_output/abalone', timestamp_prefix)\n",
    "job_name = 'serial-inference-batch-' + timestamp_prefix\n",
    "transformer = sagemaker.transformer.Transformer(\n",
    "    # This was the model created using PipelineModel and it contains feature processing and XGBoost\n",
    "    model_name = model_name,\n",
    "    instance_count = 1,\n",
    "    instance_type = 'ml.m4.xlarge',\n",
    "    strategy = 'SingleRecord',\n",
    "    assemble_with = 'Line',\n",
    "    output_path = output_data_path,\n",
    "    base_transform_job_name='serial-inference-batch',\n",
    "    sagemaker_session=sagemaker_session,\n",
    "    accept = CONTENT_TYPE_CSV\n",
    ")\n",
    "transformer.transform(data = input_data_path,\n",
    "                      job_name = job_name,\n",
    "                      content_type = CONTENT_TYPE_CSV, \n",
    "                      split_type = 'Line')\n",
    "transformer.wait()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.298088073730469\r\n",
      "7.401501178741455\r\n",
      "9.903568267822266\r\n",
      "9.100788116455078\r\n",
      "6.014784336090088\r\n",
      "7.401668548583984\r\n",
      "12.664950370788574\r\n",
      "10.9097900390625\r\n",
      "8.985713958740234\r\n",
      "11.950716972351074\r\n"
     ]
    }
   ],
   "source": [
    "s3 = boto3.resource('s3')\n",
    "\n",
    "KEY = 'batch_output/abalone/2020-06-11-18-47-41/batch_input_abalone.csv.out'\n",
    "s3.Bucket(bucket).download_file(KEY, 'batch_output_abalone.csv')\n",
    "\n",
    "!head batch_output_abalone.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}