notebooks/services/jobs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6e610b6e",
   "metadata": {},
   "source": [
    "## Jobs API"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "374d92b4",
   "metadata": {},
   "source": [
    "This API provides the possibility to create a job of an existing python or spark program, execute it and access application logs."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b79e1085",
   "metadata": {},
   "source": [
    "## Scope"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fac87590",
   "metadata": {},
   "source": [
    "* Define a job configuration\n",
    "* Create a new job\n",
    "* Execute the job\n",
    "* Download the logs\n",
    "* Update job configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c5beb4d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import hopsworks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "784247c7",
   "metadata": {},
   "source": [
    "## Connect to the cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "23ac970c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected. Call `.close()` to terminate connection gracefully.\n"
     ]
    }
   ],
   "source": [
    "# Connect to your cluster, to be used running inside Jupyter or jobs inside the cluster.\n",
    "connection = hopsworks.connection()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0c48faaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment when connecting to the cluster from an external environment.\n",
    "# connection = hopsworks.connection(project='my_project', host='my_instance', port=443, api_key_value='apikey')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b75cdf45",
   "metadata": {},
   "source": [
    "## Get the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cef9605b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the project object, if used inside your hopsworks cluster it gets the current project\n",
    "project = connection.get_project()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e288cedf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Uncomment to get specific project\n",
    "# project = connection.get_project('my_project')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c26f619",
   "metadata": {},
   "source": [
    "## Upload SparkPI program to cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "554aa4c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_api = project.get_dataset_api()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c82447aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2022-04-12 13:23:19--  https://repo.hops.works/dev/robin/spark-examples.jar\n",
      "Resolving repo.hops.works (repo.hops.works)... 144.91.119.112\n",
      "Connecting to repo.hops.works (repo.hops.works)|144.91.119.112|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1527170 (1.5M) [application/java-archive]\n",
      "Saving to: ‘spark-examples.jar.3’\n",
      "\n",
      "spark-examples.jar. 100%[===================>]   1.46M  --.-KB/s    in 0.04s   \n",
      "\n",
      "2022-04-12 13:23:19 (39.1 MB/s) - ‘spark-examples.jar.3’ saved [1527170/1527170]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Download a file to work with\n",
    "!wget https://repo.hops.works/dev/robin/spark-examples.jar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "870b44ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d940151b972641a1973874aa41c4f698",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Uploading', max=1527170.0, style=ProgressStyle(descriptio…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "uploaded_app_file = dataset_api.upload(\"spark-examples.jar\", \"Resources\", overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "567fe494",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Resources/spark-examples.jar'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uploaded_app_file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f92b8f04",
   "metadata": {},
   "source": [
    "## Get job configuration for a Spark Job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "74fc4a2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "jobs_api = project.get_jobs_api()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "141f18e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_config = jobs_api.get_configuration(\"SPARK\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3a54f863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'sparkJobConfiguration',\n",
       " 'amQueue': 'default',\n",
       " 'amMemory': 2048,\n",
       " 'amVCores': 1,\n",
       " 'spark.executor.instances': 1,\n",
       " 'spark.executor.cores': 1,\n",
       " 'spark.executor.memory': 4096,\n",
       " 'spark.executor.gpus': 0,\n",
       " 'spark.tensorflow.num.ps': 1,\n",
       " 'spark.dynamicAllocation.enabled': True,\n",
       " 'spark.dynamicAllocation.minExecutors': 1,\n",
       " 'spark.dynamicAllocation.maxExecutors': 2,\n",
       " 'spark.dynamicAllocation.initialExecutors': 1,\n",
       " 'spark.blacklist.enabled': False}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c01eeeb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Override configuration properties\n",
    "spark_config['appPath'] = uploaded_app_file\n",
    "spark_config['mainClass'] = 'org.apache.spark.examples.SparkPi'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fbca875",
   "metadata": {},
   "source": [
    "## Create a job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d6e854a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "JOB_NAME=\"my_job\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1774e0a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs_api.exists(JOB_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9c41f7e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the job and get a reference to it\n",
    "my_job = jobs_api.create_job(JOB_NAME, spark_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d9936663",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jobs_api.exists(JOB_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3df56c6f",
   "metadata": {},
   "source": [
    "## Execute the job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "07dbd973",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-04-12 13:23:21,240 INFO: Waiting for execution to finish. Current state: INITIALIZING. Final status: UNDEFINED\n",
      "2022-04-12 13:23:27,467 INFO: Waiting for execution to finish. Current state: ACCEPTED. Final status: UNDEFINED\n",
      "2022-04-12 13:23:46,149 INFO: Waiting for execution to finish. Current state: RUNNING. Final status: UNDEFINED\n",
      "2022-04-12 13:24:08,049 INFO: Waiting for execution to finish. Current state: AGGREGATING_LOGS. Final status: SUCCEEDED\n",
      "2022-04-12 13:24:11,123 INFO: Waiting for execution to finish. Current state: FINISHED. Final status: SUCCEEDED\n",
      "2022-04-12 13:24:14,126 INFO: Waiting for log aggregation to finish.\n",
      "2022-04-12 13:24:17,411 INFO: Execution finished successfully.\n"
     ]
    }
   ],
   "source": [
    "# Execute the job and optionally pass arguments.\n",
    "# If await_termination is set to True, block until execution is finished\n",
    "my_execution = my_job.run(args=\"100\", await_termination=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0d7c19e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# True or False, indicates if execution ran successfully\n",
    "my_execution.success "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68f6225d",
   "metadata": {},
   "source": [
    "## Download the logs for the execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "49be3df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "257ec2fff6974315b19d76990a165ade",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1307.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6040e4a93b6e4abaa888e4f370653b4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=153101.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Download logs and return file paths in where they exists on disk\n",
    "out, err = my_execution.download_logs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8cd70ed2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Container: container_1649425494064_0008_01_000001 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: prelaunch.out\n",
      "Log Length: 100\n",
      "Log Contents: \n",
      "Setting up env variables\n",
      "Setting up job resources\n",
      "Copying debugging information\n",
      "Launching container\n",
      "\n",
      "Log Type: stdout\n",
      "Log Length: 33\n",
      "Log Contents: \n",
      "Pi is roughly 3.1423403142340316\n",
      "\n",
      "Log Type: stdout.txt\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000003 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: prelaunch.out\n",
      "Log Length: 100\n",
      "Log Contents: \n",
      "Setting up env variables\n",
      "Setting up job resources\n",
      "Copying debugging information\n",
      "Launching container\n",
      "\n",
      "Log Type: stdout\n",
      "Log Length: 0\n",
      "Log Type: stdout.txt\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000002 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: prelaunch.out\n",
      "Log Length: 100\n",
      "Log Contents: \n",
      "Setting up env variables\n",
      "Setting up job resources\n",
      "Copying debugging information\n",
      "Launching container\n",
      "\n",
      "Log Type: stdout\n",
      "Log Length: 0\n",
      "Log Type: stdout.txt\n",
      "Log Length: 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f_out = open(out, \"r\")\n",
    "print(f_out.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "eb196022",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Container: container_1649425494064_0008_01_000001 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: prelaunch.err\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000001 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: stderr\n",
      "Log Length: 87392\n",
      "Log Contents: \n",
      "2022-04-12 13:23:29,630 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for TERM\n",
      "2022-04-12 13:23:29,632 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for HUP\n",
      "2022-04-12 13:23:29,633 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for INT\n",
      "2022-04-12 13:23:31,131 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:31,131 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:31,132 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls groups to: \n",
      "2022-04-12 13:23:31,133 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls groups to: \n",
      "2022-04-12 13:23:31,133 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with view permissions: Set(); users  with modify permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with modify permissions: Set()\n",
      "2022-04-12 13:23:32,151 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: ApplicationAttemptId: appattempt_1649425494064_0008_000001\n",
      "2022-04-12 13:23:32,249 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Starting the user application in a separate Thread\n",
      "2022-04-12 13:23:32,251 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Waiting for spark context initialization...\n",
      "2022-04-12 13:23:32,353 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkContext: Running Spark version 3.1.1.0\n",
      "2022-04-12 13:23:32,430 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: ==============================================================\n",
      "2022-04-12 13:23:32,431 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: No custom resources configured for spark.driver.\n",
      "2022-04-12 13:23:32,431 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: ==============================================================\n",
      "2022-04-12 13:23:32,433 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkContext: Submitted application: Spark Pi\n",
      "2022-04-12 13:23:32,624 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 4096, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0)\n",
      "2022-04-12 13:23:32,640 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceProfile: Limiting resource is cpus at 1 tasks per executor\n",
      "2022-04-12 13:23:32,646 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceProfileManager: Added ResourceProfile id: 0\n",
      "2022-04-12 13:23:32,937 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:32,937 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:32,938 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls groups to: \n",
      "2022-04-12 13:23:32,938 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls groups to: \n",
      "2022-04-12 13:23:32,938 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with view permissions: Set(); users  with modify permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with modify permissions: Set()\n",
      "2022-04-12 13:23:33,289 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Successfully started service 'sparkDriver' on port 35603.\n",
      "2022-04-12 13:23:33,333 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkEnv: Registering MapOutputTracker\n",
      "2022-04-12 13:23:33,391 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkEnv: Registering BlockManagerMaster\n",
      "2022-04-12 13:23:33,430 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information\n",
      "2022-04-12 13:23:33,431 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up\n",
      "2022-04-12 13:23:33,483 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkEnv: Registering BlockManagerMasterHeartbeat\n",
      "2022-04-12 13:23:33,520 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DiskBlockManager: Created local directory at /srv/hops/hopsdata/tmp/nm-local-dir/usercache/demo_ml_meb10000__meb10000/appcache/application_1649425494064_0008/blockmgr-5fcaca20-7f6a-48c0-b8d5-8a71ca24f186\n",
      "2022-04-12 13:23:33,892 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: MemoryStore started with capacity 912.3 MiB\n",
      "2022-04-12 13:23:33,974 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkEnv: Registering OutputCommitCoordinator\n",
      "2022-04-12 13:23:34,145 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 log: Logging initialized @6720ms to org.sparkproject.jetty.util.log.Slf4jLog\n",
      "2022-04-12 13:23:34,269 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Server: jetty-9.4.36.v20210114; built: 2021-01-14T16:44:28.689Z; git: 238ec6997c7806b055319a6d11f8ae7564adc0de; jvm 1.8.0_312-8u312-b07-0ubuntu1~18.04-b07\n",
      "2022-04-12 13:23:34,309 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Server: Started @6886ms\n",
      "2022-04-12 13:23:34,380 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 AbstractConnector: Started ServerConnector@68ee8eb8{HTTP/1.1, (http/1.1)}{0.0.0.0:41268}\n",
      "2022-04-12 13:23:34,380 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Successfully started service 'SparkUI' on port 41268.\n",
      "2022-04-12 13:23:34,386 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /jobs: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,444 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@6338f01d{/jobs,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,449 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /jobs/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,451 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@c01fb2f{/jobs/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,451 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /jobs/job: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,453 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@513ad5d3{/jobs/job,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,453 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /jobs/job/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,459 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@19e1ac37{/jobs/job/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,460 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,461 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@548ff507{/stages,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,462 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,464 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@4aa61252{/stages/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,464 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages/stage: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,467 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@247f1{/stages/stage,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,467 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages/stage/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,784 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@6dfbd02{/stages/stage/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,785 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages/pool: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,789 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@5138dc62{/stages/pool,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,790 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages/pool/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,792 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@3e550785{/stages/pool/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,793 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /storage: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,795 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@3644b18{/storage,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,796 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /storage/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,798 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@4586ee81{/storage/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,799 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /storage/rdd: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,801 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@385a868b{/storage/rdd,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,801 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /storage/rdd/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,805 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@434e1b86{/storage/rdd/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,805 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /environment: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,808 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@57e963d1{/environment,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,808 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /environment/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,811 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@208a1ab4{/environment/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,812 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /executors: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,821 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@5b30a151{/executors,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,822 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /executors/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,826 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@58d54a18{/executors/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,827 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /executors/threadDump: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,834 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@7e659339{/executors/threadDump,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,835 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /executors/threadDump/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,844 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@21f338b5{/executors/threadDump/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,845 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /static: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,866 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@e054c60{/static,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,866 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,868 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@318c5489{/,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,869 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /api: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,871 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@1195192c{/api,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,871 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /jobs/job/kill: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,873 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@516fca30{/jobs/job/kill,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,874 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /stages/stage/kill: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:34,876 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@4360b09e{/stages/stage/kill,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:34,879 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkUI: Bound SparkUI to 0.0.0.0, and started at http://hopsworks0.logicalclocks.com:41268\n",
      "2022-04-12 13:23:35,027 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterScheduler: Created YarnClusterScheduler\n",
      "2022-04-12 13:23:35,719 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Using initial executors = 1, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances\n",
      "2022-04-12 13:23:35,736 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 41374.\n",
      "2022-04-12 13:23:35,736 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 NettyBlockTransferService: Server created on hopsworks0.logicalclocks.com:41374\n",
      "2022-04-12 13:23:35,738 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy\n",
      "2022-04-12 13:23:35,753 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: Registering BlockManager BlockManagerId(driver, hopsworks0.logicalclocks.com, 41374, None)\n",
      "2022-04-12 13:23:35,759 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMasterEndpoint: Registering block manager hopsworks0.logicalclocks.com:41374 with 912.3 MiB RAM, BlockManagerId(driver, hopsworks0.logicalclocks.com, 41374, None)\n",
      "2022-04-12 13:23:35,763 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: Registered BlockManager BlockManagerId(driver, hopsworks0.logicalclocks.com, 41374, None)\n",
      "2022-04-12 13:23:35,764 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: external shuffle service port = 7337\n",
      "2022-04-12 13:23:35,766 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Initialized BlockManager: BlockManagerId(driver, hopsworks0.logicalclocks.com, 41374, None)\n",
      "2022-04-12 13:23:36,033 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Initializing Prometheus Sink...\n",
      "2022-04-12 13:23:36,034 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics polling period -> 10 SECONDS\n",
      "2022-04-12 13:23:36,034 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics timestamp enabled -> false\n",
      "2022-04-12 13:23:36,035 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address -> pushgateway.prometheus.service.consul:9095\n",
      "2022-04-12 13:23:36,035 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address-protocol -> http\n",
      "2022-04-12 13:23:36,036 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-capture-regex -> \n",
      "2022-04-12 13:23:36,036 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-replacement -> \n",
      "2022-04-12 13:23:36,037 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: labels -> Map()\n",
      "2022-04-12 13:23:36,037 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: jmx-collector-config -> /opt/spark/conf/jmx_collector.yaml\n",
      "2022-04-12 13:23:36,087 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ServerInfo: Adding filter to /metrics/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter\n",
      "2022-04-12 13:23:36,089 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ContextHandler: Started o.s.j.s.ServletContextHandler@589ed5d2{/metrics/json,null,AVAILABLE,@Spark}\n",
      "2022-04-12 13:23:40,963 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SingleEventLogFileWriter: Logging events to hdfs://rpc.namenode.service.consul:8020/user/spark/applicationHistory/application_1649425494064_0008_1.snappy.inprogress\n",
      "2022-04-12 13:23:41,748 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Using initial executors = 1, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances\n",
      "2022-04-12 13:23:41,773 WARN demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!\n",
      "2022-04-12 13:23:42,335 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 RMProxy: Connecting to ResourceManager at resourcemanager.service.consul/10.0.2.15:8030\n",
      "2022-04-12 13:23:42,392 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnRMClient: Registering the ApplicationMaster\n",
      "2022-04-12 13:23:42,569 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Preparing Local resources\n",
      "2022-04-12 13:23:42,751 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: \n",
      "===============================================================================\n",
      "Default YARN executor launch context:\n",
      "  env:\n",
      "    REQUESTS_VERIFY -> true\n",
      "    REST_ENDPOINT -> https://hopsworks.glassfish.service.consul:8182\n",
      "    ISTIO_ENDPOINT -> http://10.0.2.15:31699\n",
      "    HADOOP_USER_NAME -> demo_ml_meb10000__meb10000\n",
      "    CLASSPATH -> /srv/hops/hopsdata/tmp/nm-local-dir/usercache/demo_ml_meb10000__meb10000/appcache/application_1649425494064_0008/container_1649425494064_0008_01_000001:/srv/hops/spark/jars/*:/srv/hops/spark/hopsworks-jars/*<CPS>{{PWD}}<CPS>{{PWD}}/__spark_conf__<CPS>{{PWD}}/__spark_libs__/*<CPS>/srv/hops/spark/jars/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/lib/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/etc/hadoop/<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/common/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/common/lib/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/hdfs/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/hdfs/lib/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/yarn/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/yarn/lib/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/tools/lib/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/mapreduce/*<CPS>/srv/hops/hadoop-3.2.0.6-EE-RC1/share/hadoop/mapreduce/lib/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*<CPS>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*<CPS>{{PWD}}/__spark_conf__/__hadoop_conf__\n",
      "    KAFKA_VERSION -> 2.3.0\n",
      "    EXECUTOR_GPUS -> 0\n",
      "    LIVY_VERSION -> 0.8.2-incubating-SNAPSHOT-bin\n",
      "    ELASTIC_ENDPOINT -> https://10.0.2.15:9200\n",
      "    SPARK_YARN_STAGING_DIR -> /Projects/demo_ml_meb10000/Resources/.sparkjobstaging-application_1649425494064_0008\n",
      "    HOPSWORKS_PROJECT_ID -> 119\n",
      "    TENSORFLOW_VERSION -> 2.4.1\n",
      "    HADOOP_VERSION -> 3.2.0.6-EE-RC1\n",
      "    SPARK_USER -> demo_ml_meb10000__meb10000\n",
      "    DOMAIN_CA_TRUSTSTORE -> t_certificate\n",
      "    KAFKA_BROKERS -> INTERNAL://10.0.2.15:9091\n",
      "    HADOOP_HOME -> /srv/hops/hadoop\n",
      "    HADOOP_HDFS_HOME -> /srv/hops/hadoop\n",
      "    SERVING_API_KEY -> wMdMURpg4CAsgkdr.jZ8BRN68jwipwRAKwwYzClNwXh5qIHOE6zrSd5iggkWA6Ox62MW1MHsPMlI3PeFe\n",
      "    HIP_VISIBLE_DEVICES -> -1\n",
      "    SPARK_CONF_DIR -> /srv/hops/spark/conf\n",
      "    HOPSWORKS_JOB_NAME -> my_job\n",
      "    SPARK_YARN_MODE -> true\n",
      "    SPARK_HOME -> /srv/hops/spark\n",
      "    SERVICE_DISCOVERY_DOMAIN -> consul\n",
      "    YARN_CONTAINER_RUNTIME_TYPE -> docker\n",
      "    SPARK_VERSION -> 3.1.1.0\n",
      "    CUDA_VISIBLE_DEVICES -> \n",
      "    YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS -> /srv/hops/hadoop/etc/hadoop:/srv/hops/hadoop/etc/hadoop:ro,/srv/hops/spark:/srv/hops/spark:ro,/srv/hops/flink:/srv/hops/flink:ro,/srv/hops/apache-livy:/srv/hops/apache-livy:ro\n",
      "    YARN_CONTAINER_RUNTIME_DOCKER_IMAGE -> registry.service.consul:4443/demo_ml_meb10000:1649677539619-2.6.0-SNAPSHOT.4\n",
      "    HOPSWORKS_VERSION -> 2.6.0-SNAPSHOT\n",
      "    LIBHDFS_OPTS -> -Dlog4j.configuration=/srv/hops/hadoop/etc/hadoop/log4j.properties -Dhadoop.root.logger=ERROR,RFA -Xmx819m\n",
      "    PYSPARK_PYTHON -> /srv/hops/anaconda/envs/theenv/bin/python\n",
      "    FLINK_CONF_DIR -> /srv/hops/flink/conf/\n",
      "\n",
      "  command:\n",
      "    {{JAVA_HOME}}/bin/java \\ \n",
      "      -server \\ \n",
      "      -Xmx4096m \\ \n",
      "      '-Dlog4j.configuration=log4j.properties' \\ \n",
      "      '-Dhopsworks.projectname=demo_ml_meb10000' \\ \n",
      "      '-Dhopsworks.domain.truststore=t_certificate' \\ \n",
      "      '-Dhopsutil.insecure=false' \\ \n",
      "      '-Dhopsworks.logstash.job.info=demo_ml_meb10000,my_job,43,application_1649425494064_0008' \\ \n",
      "      '-Dhopsworks.elastic.endpoint=https://10.0.2.15:9200' \\ \n",
      "      '-Dhopsworks.kafka.brokeraddress=INTERNAL://10.0.2.15:9091' \\ \n",
      "      '-Dhopsworks.job.name=my_job' \\ \n",
      "      '-Dserver.truststore=server.truststore' \\ \n",
      "      '-Dhopsworks.job.appid=application_1649425494064_0008' \\ \n",
      "      '-Djava.library.path=/srv/hops/hadoop/lib/native/' \\ \n",
      "      '-Dhopsworks.projectid=119' \\ \n",
      "      '-Dhopsworks.restendpoint=https://hopsworks.glassfish.service.consul:8182' \\ \n",
      "      '-Dhopsworks.job.type=SPARK' \\ \n",
      "      '-Dhopsworks.projectuser=demo_ml_meb10000__meb10000' \\ \n",
      "      -Djava.io.tmpdir={{PWD}}/tmp \\ \n",
      "      '-Dspark.driver.port=35603' \\ \n",
      "      '-Dspark.ui.port=0' \\ \n",
      "      -Dspark.yarn.app.container.log.dir=<LOG_DIR> \\ \n",
      "      -XX:OnOutOfMemoryError='kill %p' \\ \n",
      "      org.apache.spark.executor.YarnCoarseGrainedExecutorBackend \\ \n",
      "      --driver-url \\ \n",
      "      spark://CoarseGrainedScheduler@hopsworks0.logicalclocks.com:35603 \\ \n",
      "      --executor-id \\ \n",
      "      <executorId> \\ \n",
      "      --hostname \\ \n",
      "      <hostname> \\ \n",
      "      --cores \\ \n",
      "      1 \\ \n",
      "      --app-id \\ \n",
      "      application_1649425494064_0008 \\ \n",
      "      --resourceProfileId \\ \n",
      "      0 \\ \n",
      "      --user-class-path \\ \n",
      "      file:$PWD/__app__.jar \\ \n",
      "      1><LOG_DIR>/stdout \\ \n",
      "      2><LOG_DIR>/stderr\n",
      "\n",
      "  resources:\n",
      "    log4j.properties -> resource { scheme: \"hdfs\" port: -1 file: \"/user/spark/log4j.properties\" } size: 943 timestamp: 1649425715270 type: FILE visibility: APPLICATION\n",
      "    __app__.jar -> resource { scheme: \"hdfs\" port: -1 file: \"/Projects/demo_ml_meb10000/Resources/spark-examples.jar\" } size: 1527170 timestamp: 1649769800590 type: FILE visibility: APPLICATION\n",
      "\n",
      "===============================================================================\n",
      "2022-04-12 13:23:43,144 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Using initial executors = 1, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances\n",
      "2022-04-12 13:23:43,146 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Resource profile 0 doesn't exist, adding it\n",
      "2022-04-12 13:23:43,195 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Configuration: found resource resource-types.xml at file:/srv/hops/hadoop-3.2.0.6-EE-RC1/etc/hadoop/resource-types.xml\n",
      "2022-04-12 13:23:43,215 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: Adding resource type - name = yarn.io/gpu, units = , type = COUNTABLE\n",
      "2022-04-12 13:23:43,239 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnSchedulerBackend$YarnSchedulerEndpoint: ApplicationMaster registered as NettyRpcEndpointRef(spark://YarnAM@hopsworks0.logicalclocks.com:35603)\n",
      "2022-04-12 13:23:43,257 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Will request 1 executor container(s) for  ResourceProfile Id: 0, each with 1 core(s) and 4505 MB memory. with custom resources: <memory:4505, vCores:1>\n",
      "2022-04-12 13:23:43,290 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Submitted 1 unlocalized container requests.\n",
      "2022-04-12 13:23:43,381 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Initializing Prometheus Sink...\n",
      "2022-04-12 13:23:43,382 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics polling period -> 10 SECONDS\n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics timestamp enabled -> false\n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address -> pushgateway.prometheus.service.consul:9095\n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address-protocol -> http\n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-capture-regex -> \n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-replacement -> \n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: labels -> Map()\n",
      "2022-04-12 13:23:43,383 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: jmx-collector-config -> /opt/spark/conf/jmx_collector.yaml\n",
      "2022-04-12 13:23:43,392 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Started progress reporter thread with (heartbeat : 5000, initial allocation : 200) intervals\n",
      "2022-04-12 13:23:43,644 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Launching container container_1649425494064_0008_01_000002 on host hopsworks0.logicalclocks.com for executor with ID 1 for ResourceProfile Id 0\n",
      "2022-04-12 13:23:43,650 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Received 1 containers from YARN, launching executors on 1 of them.\n",
      "2022-04-12 13:23:46,085 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(driver)\n",
      "2022-04-12 13:23:46,086 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=driver, job=application_1649425494064_0008\n",
      "2022-04-12 13:23:51,118 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnSchedulerBackend$YarnDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.0.2.15:32776) with ID 1,  ResourceProfileId 0\n",
      "2022-04-12 13:23:51,144 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ExecutorMonitor: New executor 1 has registered (new total is 1)\n",
      "2022-04-12 13:23:51,150 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.8\n",
      "2022-04-12 13:23:51,151 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterScheduler: YarnClusterScheduler.postStartHook done\n",
      "2022-04-12 13:23:51,596 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMasterEndpoint: Registering block manager hopsworks0.logicalclocks.com:40427 with 2004.6 MiB RAM, BlockManagerId(1, hopsworks0.logicalclocks.com, 40427, None)\n",
      "2022-04-12 13:23:51,731 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkContext: Starting job: reduce at SparkPi.scala:38\n",
      "2022-04-12 13:23:51,756 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Got job 0 (reduce at SparkPi.scala:38) with 100 output partitions\n",
      "2022-04-12 13:23:51,757 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Final stage: ResultStage 0 (reduce at SparkPi.scala:38)\n",
      "2022-04-12 13:23:51,757 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Parents of final stage: List()\n",
      "2022-04-12 13:23:51,759 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Missing parents: List()\n",
      "2022-04-12 13:23:51,770 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34), which has no missing parents\n",
      "2022-04-12 13:23:51,885 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Resource profile 0 doesn't exist, adding it\n",
      "2022-04-12 13:23:51,886 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Driver requested a total number of 2 executor(s) for resource profile id: 0.\n",
      "2022-04-12 13:23:51,892 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Will request 1 executor container(s) for  ResourceProfile Id: 0, each with 1 core(s) and 4505 MB memory. with custom resources: <memory:4505, vCores:1>\n",
      "2022-04-12 13:23:51,893 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Submitted 1 unlocalized container requests.\n",
      "2022-04-12 13:23:51,895 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ExecutorAllocationManager: Requesting 1 new executor because tasks are backlogged (new desired total will be 2 for resource profile id: 0)\n",
      "2022-04-12 13:23:51,930 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: Block broadcast_0 stored as values in memory (estimated size 3.1 KiB, free 912.3 MiB)\n",
      "2022-04-12 13:23:52,802 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Launching container container_1649425494064_0008_01_000003 on host hopsworks0.logicalclocks.com for executor with ID 2 for ResourceProfile Id 0\n",
      "2022-04-12 13:23:52,804 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Received 1 containers from YARN, launching executors on 1 of them.\n",
      "2022-04-12 13:23:52,826 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1736.0 B, free 912.3 MiB)\n",
      "2022-04-12 13:23:52,833 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerInfo: Added broadcast_0_piece0 in memory on hopsworks0.logicalclocks.com:41374 (size: 1736.0 B, free: 912.3 MiB)\n",
      "2022-04-12 13:23:52,838 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1383\n",
      "2022-04-12 13:23:52,878 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Submitting 100 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34) (first 15 tasks are for partitions Vector(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))\n",
      "2022-04-12 13:23:52,879 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterScheduler: Adding task set 0.0 with 100 tasks resource profile 0\n",
      "2022-04-12 13:23:52,973 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0) (hopsworks0.logicalclocks.com, executor 1, partition 0, PROCESS_LOCAL, 4324 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:53,394 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(driver)\n",
      "2022-04-12 13:23:53,394 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=driver, job=application_1649425494064_0008\n",
      "2022-04-12 13:23:54,597 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerInfo: Added broadcast_0_piece0 in memory on hopsworks0.logicalclocks.com:40427 (size: 1736.0 B, free: 2004.6 MiB)\n",
      "2022-04-12 13:23:56,081 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(driver)\n",
      "2022-04-12 13:23:56,082 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=driver, job=application_1649425494064_0008\n",
      "2022-04-12 13:23:56,239 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1) (hopsworks0.logicalclocks.com, executor 1, partition 1, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,265 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 3316 ms on hopsworks0.logicalclocks.com (executor 1) (1/100)\n",
      "2022-04-12 13:23:56,320 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 2.0 in stage 0.0 (TID 2) (hopsworks0.logicalclocks.com, executor 1, partition 2, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,323 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 87 ms on hopsworks0.logicalclocks.com (executor 1) (2/100)\n",
      "2022-04-12 13:23:56,397 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 3.0 in stage 0.0 (TID 3) (hopsworks0.logicalclocks.com, executor 1, partition 3, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,398 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 2.0 in stage 0.0 (TID 2) in 77 ms on hopsworks0.logicalclocks.com (executor 1) (3/100)\n",
      "2022-04-12 13:23:56,444 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 4.0 in stage 0.0 (TID 4) (hopsworks0.logicalclocks.com, executor 1, partition 4, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,447 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 3.0 in stage 0.0 (TID 3) in 51 ms on hopsworks0.logicalclocks.com (executor 1) (4/100)\n",
      "2022-04-12 13:23:56,476 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 5.0 in stage 0.0 (TID 5) (hopsworks0.logicalclocks.com, executor 1, partition 5, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,478 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 4.0 in stage 0.0 (TID 4) in 35 ms on hopsworks0.logicalclocks.com (executor 1) (5/100)\n",
      "2022-04-12 13:23:56,525 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 6.0 in stage 0.0 (TID 6) (hopsworks0.logicalclocks.com, executor 1, partition 6, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,526 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 5.0 in stage 0.0 (TID 5) in 50 ms on hopsworks0.logicalclocks.com (executor 1) (6/100)\n",
      "2022-04-12 13:23:56,553 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 7.0 in stage 0.0 (TID 7) (hopsworks0.logicalclocks.com, executor 1, partition 7, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,554 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 6.0 in stage 0.0 (TID 6) in 30 ms on hopsworks0.logicalclocks.com (executor 1) (7/100)\n",
      "2022-04-12 13:23:56,596 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 8.0 in stage 0.0 (TID 8) (hopsworks0.logicalclocks.com, executor 1, partition 8, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,597 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 7.0 in stage 0.0 (TID 7) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (8/100)\n",
      "2022-04-12 13:23:56,640 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 9.0 in stage 0.0 (TID 9) (hopsworks0.logicalclocks.com, executor 1, partition 9, PROCESS_LOCAL, 4326 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,641 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 8.0 in stage 0.0 (TID 8) in 46 ms on hopsworks0.logicalclocks.com (executor 1) (9/100)\n",
      "2022-04-12 13:23:56,691 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 10.0 in stage 0.0 (TID 10) (hopsworks0.logicalclocks.com, executor 1, partition 10, PROCESS_LOCAL, 4328 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,692 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 9.0 in stage 0.0 (TID 9) in 53 ms on hopsworks0.logicalclocks.com (executor 1) (10/100)\n",
      "2022-04-12 13:23:56,737 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 11.0 in stage 0.0 (TID 11) (hopsworks0.logicalclocks.com, executor 1, partition 11, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,739 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 10.0 in stage 0.0 (TID 10) in 48 ms on hopsworks0.logicalclocks.com (executor 1) (11/100)\n",
      "2022-04-12 13:23:56,772 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 12.0 in stage 0.0 (TID 12) (hopsworks0.logicalclocks.com, executor 1, partition 12, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,773 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 11.0 in stage 0.0 (TID 11) in 36 ms on hopsworks0.logicalclocks.com (executor 1) (12/100)\n",
      "2022-04-12 13:23:56,799 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 13.0 in stage 0.0 (TID 13) (hopsworks0.logicalclocks.com, executor 1, partition 13, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,800 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 12.0 in stage 0.0 (TID 12) in 29 ms on hopsworks0.logicalclocks.com (executor 1) (13/100)\n",
      "2022-04-12 13:23:56,830 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 14.0 in stage 0.0 (TID 14) (hopsworks0.logicalclocks.com, executor 1, partition 14, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,834 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 13.0 in stage 0.0 (TID 13) in 34 ms on hopsworks0.logicalclocks.com (executor 1) (14/100)\n",
      "2022-04-12 13:23:56,858 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 15.0 in stage 0.0 (TID 15) (hopsworks0.logicalclocks.com, executor 1, partition 15, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,860 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 14.0 in stage 0.0 (TID 14) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (15/100)\n",
      "2022-04-12 13:23:56,892 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 16.0 in stage 0.0 (TID 16) (hopsworks0.logicalclocks.com, executor 1, partition 16, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,893 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 15.0 in stage 0.0 (TID 15) in 37 ms on hopsworks0.logicalclocks.com (executor 1) (16/100)\n",
      "2022-04-12 13:23:56,919 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 17.0 in stage 0.0 (TID 17) (hopsworks0.logicalclocks.com, executor 1, partition 17, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,920 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 16.0 in stage 0.0 (TID 16) in 29 ms on hopsworks0.logicalclocks.com (executor 1) (17/100)\n",
      "2022-04-12 13:23:56,942 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 18.0 in stage 0.0 (TID 18) (hopsworks0.logicalclocks.com, executor 1, partition 18, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,943 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 17.0 in stage 0.0 (TID 17) in 25 ms on hopsworks0.logicalclocks.com (executor 1) (18/100)\n",
      "2022-04-12 13:23:56,972 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 19.0 in stage 0.0 (TID 19) (hopsworks0.logicalclocks.com, executor 1, partition 19, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:56,973 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 18.0 in stage 0.0 (TID 18) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (19/100)\n",
      "2022-04-12 13:23:57,015 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 20.0 in stage 0.0 (TID 20) (hopsworks0.logicalclocks.com, executor 1, partition 20, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,016 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 19.0 in stage 0.0 (TID 19) in 46 ms on hopsworks0.logicalclocks.com (executor 1) (20/100)\n",
      "2022-04-12 13:23:57,046 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 21.0 in stage 0.0 (TID 21) (hopsworks0.logicalclocks.com, executor 1, partition 21, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,048 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 20.0 in stage 0.0 (TID 20) in 33 ms on hopsworks0.logicalclocks.com (executor 1) (21/100)\n",
      "2022-04-12 13:23:57,093 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 22.0 in stage 0.0 (TID 22) (hopsworks0.logicalclocks.com, executor 1, partition 22, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,101 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 21.0 in stage 0.0 (TID 21) in 56 ms on hopsworks0.logicalclocks.com (executor 1) (22/100)\n",
      "2022-04-12 13:23:57,141 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 23.0 in stage 0.0 (TID 23) (hopsworks0.logicalclocks.com, executor 1, partition 23, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,142 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 22.0 in stage 0.0 (TID 22) in 50 ms on hopsworks0.logicalclocks.com (executor 1) (23/100)\n",
      "2022-04-12 13:23:57,182 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 24.0 in stage 0.0 (TID 24) (hopsworks0.logicalclocks.com, executor 1, partition 24, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,184 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 23.0 in stage 0.0 (TID 23) in 44 ms on hopsworks0.logicalclocks.com (executor 1) (24/100)\n",
      "2022-04-12 13:23:57,215 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 25.0 in stage 0.0 (TID 25) (hopsworks0.logicalclocks.com, executor 1, partition 25, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,217 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 24.0 in stage 0.0 (TID 24) in 36 ms on hopsworks0.logicalclocks.com (executor 1) (25/100)\n",
      "2022-04-12 13:23:57,254 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 26.0 in stage 0.0 (TID 26) (hopsworks0.logicalclocks.com, executor 1, partition 26, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,258 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 25.0 in stage 0.0 (TID 25) in 42 ms on hopsworks0.logicalclocks.com (executor 1) (26/100)\n",
      "2022-04-12 13:23:57,316 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 27.0 in stage 0.0 (TID 27) (hopsworks0.logicalclocks.com, executor 1, partition 27, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,321 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 26.0 in stage 0.0 (TID 26) in 68 ms on hopsworks0.logicalclocks.com (executor 1) (27/100)\n",
      "2022-04-12 13:23:57,364 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 28.0 in stage 0.0 (TID 28) (hopsworks0.logicalclocks.com, executor 1, partition 28, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,372 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 27.0 in stage 0.0 (TID 27) in 56 ms on hopsworks0.logicalclocks.com (executor 1) (28/100)\n",
      "2022-04-12 13:23:57,421 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 29.0 in stage 0.0 (TID 29) (hopsworks0.logicalclocks.com, executor 1, partition 29, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,425 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 28.0 in stage 0.0 (TID 28) in 61 ms on hopsworks0.logicalclocks.com (executor 1) (29/100)\n",
      "2022-04-12 13:23:57,457 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 30.0 in stage 0.0 (TID 30) (hopsworks0.logicalclocks.com, executor 1, partition 30, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,458 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 29.0 in stage 0.0 (TID 29) in 37 ms on hopsworks0.logicalclocks.com (executor 1) (30/100)\n",
      "2022-04-12 13:23:57,488 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 31.0 in stage 0.0 (TID 31) (hopsworks0.logicalclocks.com, executor 1, partition 31, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,495 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 30.0 in stage 0.0 (TID 30) in 38 ms on hopsworks0.logicalclocks.com (executor 1) (31/100)\n",
      "2022-04-12 13:23:57,526 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 32.0 in stage 0.0 (TID 32) (hopsworks0.logicalclocks.com, executor 1, partition 32, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,530 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 31.0 in stage 0.0 (TID 31) in 43 ms on hopsworks0.logicalclocks.com (executor 1) (32/100)\n",
      "2022-04-12 13:23:57,561 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 33.0 in stage 0.0 (TID 33) (hopsworks0.logicalclocks.com, executor 1, partition 33, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,561 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 32.0 in stage 0.0 (TID 32) in 36 ms on hopsworks0.logicalclocks.com (executor 1) (33/100)\n",
      "2022-04-12 13:23:57,606 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 34.0 in stage 0.0 (TID 34) (hopsworks0.logicalclocks.com, executor 1, partition 34, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,607 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 33.0 in stage 0.0 (TID 33) in 47 ms on hopsworks0.logicalclocks.com (executor 1) (34/100)\n",
      "2022-04-12 13:23:57,635 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 35.0 in stage 0.0 (TID 35) (hopsworks0.logicalclocks.com, executor 1, partition 35, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,637 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 34.0 in stage 0.0 (TID 34) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (35/100)\n",
      "2022-04-12 13:23:57,671 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 36.0 in stage 0.0 (TID 36) (hopsworks0.logicalclocks.com, executor 1, partition 36, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,672 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 35.0 in stage 0.0 (TID 35) in 37 ms on hopsworks0.logicalclocks.com (executor 1) (36/100)\n",
      "2022-04-12 13:23:57,737 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 37.0 in stage 0.0 (TID 37) (hopsworks0.logicalclocks.com, executor 1, partition 37, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,742 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 36.0 in stage 0.0 (TID 36) in 72 ms on hopsworks0.logicalclocks.com (executor 1) (37/100)\n",
      "2022-04-12 13:23:57,815 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 38.0 in stage 0.0 (TID 38) (hopsworks0.logicalclocks.com, executor 1, partition 38, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,818 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 37.0 in stage 0.0 (TID 37) in 81 ms on hopsworks0.logicalclocks.com (executor 1) (38/100)\n",
      "2022-04-12 13:23:57,855 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 39.0 in stage 0.0 (TID 39) (hopsworks0.logicalclocks.com, executor 1, partition 39, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,857 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 38.0 in stage 0.0 (TID 38) in 43 ms on hopsworks0.logicalclocks.com (executor 1) (39/100)\n",
      "2022-04-12 13:23:57,889 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 40.0 in stage 0.0 (TID 40) (hopsworks0.logicalclocks.com, executor 1, partition 40, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,890 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 39.0 in stage 0.0 (TID 39) in 35 ms on hopsworks0.logicalclocks.com (executor 1) (40/100)\n",
      "2022-04-12 13:23:57,951 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 41.0 in stage 0.0 (TID 41) (hopsworks0.logicalclocks.com, executor 1, partition 41, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:57,959 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 40.0 in stage 0.0 (TID 40) in 71 ms on hopsworks0.logicalclocks.com (executor 1) (41/100)\n",
      "2022-04-12 13:23:58,000 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 42.0 in stage 0.0 (TID 42) (hopsworks0.logicalclocks.com, executor 1, partition 42, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,001 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 41.0 in stage 0.0 (TID 41) in 55 ms on hopsworks0.logicalclocks.com (executor 1) (42/100)\n",
      "2022-04-12 13:23:58,077 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 43.0 in stage 0.0 (TID 43) (hopsworks0.logicalclocks.com, executor 1, partition 43, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,078 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 42.0 in stage 0.0 (TID 42) in 79 ms on hopsworks0.logicalclocks.com (executor 1) (43/100)\n",
      "2022-04-12 13:23:58,141 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 44.0 in stage 0.0 (TID 44) (hopsworks0.logicalclocks.com, executor 1, partition 44, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,143 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 43.0 in stage 0.0 (TID 43) in 66 ms on hopsworks0.logicalclocks.com (executor 1) (44/100)\n",
      "2022-04-12 13:23:58,193 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 45.0 in stage 0.0 (TID 45) (hopsworks0.logicalclocks.com, executor 1, partition 45, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,196 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 44.0 in stage 0.0 (TID 44) in 55 ms on hopsworks0.logicalclocks.com (executor 1) (45/100)\n",
      "2022-04-12 13:23:58,247 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 46.0 in stage 0.0 (TID 46) (hopsworks0.logicalclocks.com, executor 1, partition 46, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,249 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 45.0 in stage 0.0 (TID 45) in 56 ms on hopsworks0.logicalclocks.com (executor 1) (46/100)\n",
      "2022-04-12 13:23:58,316 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 47.0 in stage 0.0 (TID 47) (hopsworks0.logicalclocks.com, executor 1, partition 47, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,319 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 46.0 in stage 0.0 (TID 46) in 73 ms on hopsworks0.logicalclocks.com (executor 1) (47/100)\n",
      "2022-04-12 13:23:58,438 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 48.0 in stage 0.0 (TID 48) (hopsworks0.logicalclocks.com, executor 1, partition 48, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,440 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 47.0 in stage 0.0 (TID 47) in 125 ms on hopsworks0.logicalclocks.com (executor 1) (48/100)\n",
      "2022-04-12 13:23:58,481 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 49.0 in stage 0.0 (TID 49) (hopsworks0.logicalclocks.com, executor 1, partition 49, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,483 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 48.0 in stage 0.0 (TID 48) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (49/100)\n",
      "2022-04-12 13:23:58,511 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 50.0 in stage 0.0 (TID 50) (hopsworks0.logicalclocks.com, executor 1, partition 50, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,512 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 49.0 in stage 0.0 (TID 49) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (50/100)\n",
      "2022-04-12 13:23:58,888 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 51.0 in stage 0.0 (TID 51) (hopsworks0.logicalclocks.com, executor 1, partition 51, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,889 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 50.0 in stage 0.0 (TID 50) in 378 ms on hopsworks0.logicalclocks.com (executor 1) (51/100)\n",
      "2022-04-12 13:23:58,921 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 52.0 in stage 0.0 (TID 52) (hopsworks0.logicalclocks.com, executor 1, partition 52, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,924 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 51.0 in stage 0.0 (TID 51) in 36 ms on hopsworks0.logicalclocks.com (executor 1) (52/100)\n",
      "2022-04-12 13:23:58,965 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 53.0 in stage 0.0 (TID 53) (hopsworks0.logicalclocks.com, executor 1, partition 53, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:58,965 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 52.0 in stage 0.0 (TID 52) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (53/100)\n",
      "2022-04-12 13:23:59,008 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 54.0 in stage 0.0 (TID 54) (hopsworks0.logicalclocks.com, executor 1, partition 54, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,012 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 53.0 in stage 0.0 (TID 53) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (54/100)\n",
      "2022-04-12 13:23:59,038 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 55.0 in stage 0.0 (TID 55) (hopsworks0.logicalclocks.com, executor 1, partition 55, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,040 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 54.0 in stage 0.0 (TID 54) in 33 ms on hopsworks0.logicalclocks.com (executor 1) (55/100)\n",
      "2022-04-12 13:23:59,067 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 56.0 in stage 0.0 (TID 56) (hopsworks0.logicalclocks.com, executor 1, partition 56, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,072 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 55.0 in stage 0.0 (TID 55) in 35 ms on hopsworks0.logicalclocks.com (executor 1) (56/100)\n",
      "2022-04-12 13:23:59,090 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 57.0 in stage 0.0 (TID 57) (hopsworks0.logicalclocks.com, executor 1, partition 57, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,090 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 56.0 in stage 0.0 (TID 56) in 24 ms on hopsworks0.logicalclocks.com (executor 1) (57/100)\n",
      "2022-04-12 13:23:59,115 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 58.0 in stage 0.0 (TID 58) (hopsworks0.logicalclocks.com, executor 1, partition 58, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,117 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 57.0 in stage 0.0 (TID 57) in 27 ms on hopsworks0.logicalclocks.com (executor 1) (58/100)\n",
      "2022-04-12 13:23:59,143 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 59.0 in stage 0.0 (TID 59) (hopsworks0.logicalclocks.com, executor 1, partition 59, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,148 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 58.0 in stage 0.0 (TID 58) in 34 ms on hopsworks0.logicalclocks.com (executor 1) (59/100)\n",
      "2022-04-12 13:23:59,182 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 60.0 in stage 0.0 (TID 60) (hopsworks0.logicalclocks.com, executor 1, partition 60, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,187 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 59.0 in stage 0.0 (TID 59) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (60/100)\n",
      "2022-04-12 13:23:59,242 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 61.0 in stage 0.0 (TID 61) (hopsworks0.logicalclocks.com, executor 1, partition 61, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,243 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 60.0 in stage 0.0 (TID 60) in 62 ms on hopsworks0.logicalclocks.com (executor 1) (61/100)\n",
      "2022-04-12 13:23:59,270 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 62.0 in stage 0.0 (TID 62) (hopsworks0.logicalclocks.com, executor 1, partition 62, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,271 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 61.0 in stage 0.0 (TID 61) in 30 ms on hopsworks0.logicalclocks.com (executor 1) (62/100)\n",
      "2022-04-12 13:23:59,303 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 63.0 in stage 0.0 (TID 63) (hopsworks0.logicalclocks.com, executor 1, partition 63, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,304 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 62.0 in stage 0.0 (TID 62) in 34 ms on hopsworks0.logicalclocks.com (executor 1) (63/100)\n",
      "2022-04-12 13:23:59,343 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 64.0 in stage 0.0 (TID 64) (hopsworks0.logicalclocks.com, executor 1, partition 64, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,344 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 63.0 in stage 0.0 (TID 63) in 41 ms on hopsworks0.logicalclocks.com (executor 1) (64/100)\n",
      "2022-04-12 13:23:59,380 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 65.0 in stage 0.0 (TID 65) (hopsworks0.logicalclocks.com, executor 1, partition 65, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,382 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 64.0 in stage 0.0 (TID 64) in 40 ms on hopsworks0.logicalclocks.com (executor 1) (65/100)\n",
      "2022-04-12 13:23:59,430 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 66.0 in stage 0.0 (TID 66) (hopsworks0.logicalclocks.com, executor 1, partition 66, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,433 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 65.0 in stage 0.0 (TID 65) in 53 ms on hopsworks0.logicalclocks.com (executor 1) (66/100)\n",
      "2022-04-12 13:23:59,473 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 67.0 in stage 0.0 (TID 67) (hopsworks0.logicalclocks.com, executor 1, partition 67, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,478 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 66.0 in stage 0.0 (TID 66) in 48 ms on hopsworks0.logicalclocks.com (executor 1) (67/100)\n",
      "2022-04-12 13:23:59,804 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 68.0 in stage 0.0 (TID 68) (hopsworks0.logicalclocks.com, executor 1, partition 68, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,804 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 67.0 in stage 0.0 (TID 67) in 331 ms on hopsworks0.logicalclocks.com (executor 1) (68/100)\n",
      "2022-04-12 13:23:59,836 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 69.0 in stage 0.0 (TID 69) (hopsworks0.logicalclocks.com, executor 1, partition 69, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,848 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 68.0 in stage 0.0 (TID 68) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (69/100)\n",
      "2022-04-12 13:23:59,879 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 70.0 in stage 0.0 (TID 70) (hopsworks0.logicalclocks.com, executor 1, partition 70, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,880 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 69.0 in stage 0.0 (TID 69) in 45 ms on hopsworks0.logicalclocks.com (executor 1) (70/100)\n",
      "2022-04-12 13:23:59,920 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 71.0 in stage 0.0 (TID 71) (hopsworks0.logicalclocks.com, executor 1, partition 71, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,922 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 70.0 in stage 0.0 (TID 70) in 44 ms on hopsworks0.logicalclocks.com (executor 1) (71/100)\n",
      "2022-04-12 13:23:59,948 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 72.0 in stage 0.0 (TID 72) (hopsworks0.logicalclocks.com, executor 1, partition 72, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,951 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 71.0 in stage 0.0 (TID 71) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (72/100)\n",
      "2022-04-12 13:23:59,976 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 73.0 in stage 0.0 (TID 73) (hopsworks0.logicalclocks.com, executor 1, partition 73, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:23:59,977 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 72.0 in stage 0.0 (TID 72) in 30 ms on hopsworks0.logicalclocks.com (executor 1) (73/100)\n",
      "2022-04-12 13:24:00,034 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 74.0 in stage 0.0 (TID 74) (hopsworks0.logicalclocks.com, executor 1, partition 74, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,035 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 73.0 in stage 0.0 (TID 73) in 59 ms on hopsworks0.logicalclocks.com (executor 1) (74/100)\n",
      "2022-04-12 13:24:00,061 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 75.0 in stage 0.0 (TID 75) (hopsworks0.logicalclocks.com, executor 1, partition 75, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,063 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 74.0 in stage 0.0 (TID 74) in 29 ms on hopsworks0.logicalclocks.com (executor 1) (75/100)\n",
      "2022-04-12 13:24:00,091 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 76.0 in stage 0.0 (TID 76) (hopsworks0.logicalclocks.com, executor 1, partition 76, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,093 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 75.0 in stage 0.0 (TID 75) in 32 ms on hopsworks0.logicalclocks.com (executor 1) (76/100)\n",
      "2022-04-12 13:24:00,113 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 77.0 in stage 0.0 (TID 77) (hopsworks0.logicalclocks.com, executor 1, partition 77, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,114 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 76.0 in stage 0.0 (TID 76) in 22 ms on hopsworks0.logicalclocks.com (executor 1) (77/100)\n",
      "2022-04-12 13:24:00,139 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnSchedulerBackend$YarnDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.0.2.15:32962) with ID 2,  ResourceProfileId 0\n",
      "2022-04-12 13:24:00,143 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ExecutorMonitor: New executor 2 has registered (new total is 3)\n",
      "2022-04-12 13:24:00,151 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 78.0 in stage 0.0 (TID 78) (hopsworks0.logicalclocks.com, executor 1, partition 78, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,160 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 77.0 in stage 0.0 (TID 77) in 48 ms on hopsworks0.logicalclocks.com (executor 1) (78/100)\n",
      "2022-04-12 13:24:00,176 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 79.0 in stage 0.0 (TID 79) (hopsworks0.logicalclocks.com, executor 1, partition 79, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,177 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 78.0 in stage 0.0 (TID 78) in 26 ms on hopsworks0.logicalclocks.com (executor 1) (79/100)\n",
      "2022-04-12 13:24:00,199 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 80.0 in stage 0.0 (TID 80) (hopsworks0.logicalclocks.com, executor 1, partition 80, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,201 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 79.0 in stage 0.0 (TID 79) in 26 ms on hopsworks0.logicalclocks.com (executor 1) (80/100)\n",
      "2022-04-12 13:24:00,221 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 81.0 in stage 0.0 (TID 81) (hopsworks0.logicalclocks.com, executor 1, partition 81, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,222 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 80.0 in stage 0.0 (TID 80) in 24 ms on hopsworks0.logicalclocks.com (executor 1) (81/100)\n",
      "2022-04-12 13:24:00,242 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 82.0 in stage 0.0 (TID 82) (hopsworks0.logicalclocks.com, executor 1, partition 82, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,243 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 81.0 in stage 0.0 (TID 81) in 23 ms on hopsworks0.logicalclocks.com (executor 1) (82/100)\n",
      "2022-04-12 13:24:00,270 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 83.0 in stage 0.0 (TID 83) (hopsworks0.logicalclocks.com, executor 1, partition 83, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,274 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 82.0 in stage 0.0 (TID 82) in 33 ms on hopsworks0.logicalclocks.com (executor 1) (83/100)\n",
      "2022-04-12 13:24:00,296 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 84.0 in stage 0.0 (TID 84) (hopsworks0.logicalclocks.com, executor 1, partition 84, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,298 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 83.0 in stage 0.0 (TID 83) in 28 ms on hopsworks0.logicalclocks.com (executor 1) (84/100)\n",
      "2022-04-12 13:24:00,318 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 85.0 in stage 0.0 (TID 85) (hopsworks0.logicalclocks.com, executor 1, partition 85, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,321 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 84.0 in stage 0.0 (TID 84) in 25 ms on hopsworks0.logicalclocks.com (executor 1) (85/100)\n",
      "2022-04-12 13:24:00,345 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 86.0 in stage 0.0 (TID 86) (hopsworks0.logicalclocks.com, executor 1, partition 86, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,346 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 85.0 in stage 0.0 (TID 85) in 28 ms on hopsworks0.logicalclocks.com (executor 1) (86/100)\n",
      "2022-04-12 13:24:00,372 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 87.0 in stage 0.0 (TID 87) (hopsworks0.logicalclocks.com, executor 1, partition 87, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,373 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 86.0 in stage 0.0 (TID 86) in 29 ms on hopsworks0.logicalclocks.com (executor 1) (87/100)\n",
      "2022-04-12 13:24:00,398 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 88.0 in stage 0.0 (TID 88) (hopsworks0.logicalclocks.com, executor 1, partition 88, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,398 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 87.0 in stage 0.0 (TID 87) in 26 ms on hopsworks0.logicalclocks.com (executor 1) (88/100)\n",
      "2022-04-12 13:24:00,408 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMasterEndpoint: Registering block manager hopsworks0.logicalclocks.com:37058 with 2004.6 MiB RAM, BlockManagerId(2, hopsworks0.logicalclocks.com, 37058, None)\n",
      "2022-04-12 13:24:00,512 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 89.0 in stage 0.0 (TID 89) (hopsworks0.logicalclocks.com, executor 2, partition 89, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,785 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 90.0 in stage 0.0 (TID 90) (hopsworks0.logicalclocks.com, executor 1, partition 90, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,789 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 88.0 in stage 0.0 (TID 88) in 392 ms on hopsworks0.logicalclocks.com (executor 1) (89/100)\n",
      "2022-04-12 13:24:00,815 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 91.0 in stage 0.0 (TID 91) (hopsworks0.logicalclocks.com, executor 1, partition 91, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,816 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 90.0 in stage 0.0 (TID 90) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (90/100)\n",
      "2022-04-12 13:24:00,848 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 92.0 in stage 0.0 (TID 92) (hopsworks0.logicalclocks.com, executor 1, partition 92, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,851 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 91.0 in stage 0.0 (TID 91) in 36 ms on hopsworks0.logicalclocks.com (executor 1) (91/100)\n",
      "2022-04-12 13:24:00,872 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 93.0 in stage 0.0 (TID 93) (hopsworks0.logicalclocks.com, executor 1, partition 93, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,873 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 92.0 in stage 0.0 (TID 92) in 25 ms on hopsworks0.logicalclocks.com (executor 1) (92/100)\n",
      "2022-04-12 13:24:00,910 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 94.0 in stage 0.0 (TID 94) (hopsworks0.logicalclocks.com, executor 1, partition 94, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,912 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 93.0 in stage 0.0 (TID 93) in 41 ms on hopsworks0.logicalclocks.com (executor 1) (93/100)\n",
      "2022-04-12 13:24:00,931 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 95.0 in stage 0.0 (TID 95) (hopsworks0.logicalclocks.com, executor 1, partition 95, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,932 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 94.0 in stage 0.0 (TID 94) in 22 ms on hopsworks0.logicalclocks.com (executor 1) (94/100)\n",
      "2022-04-12 13:24:00,964 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 96.0 in stage 0.0 (TID 96) (hopsworks0.logicalclocks.com, executor 1, partition 96, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,965 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 95.0 in stage 0.0 (TID 95) in 34 ms on hopsworks0.logicalclocks.com (executor 1) (95/100)\n",
      "2022-04-12 13:24:00,986 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 97.0 in stage 0.0 (TID 97) (hopsworks0.logicalclocks.com, executor 1, partition 97, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:00,987 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 96.0 in stage 0.0 (TID 96) in 23 ms on hopsworks0.logicalclocks.com (executor 1) (96/100)\n",
      "2022-04-12 13:24:01,010 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 98.0 in stage 0.0 (TID 98) (hopsworks0.logicalclocks.com, executor 1, partition 98, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:01,017 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 97.0 in stage 0.0 (TID 97) in 31 ms on hopsworks0.logicalclocks.com (executor 1) (97/100)\n",
      "2022-04-12 13:24:01,041 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Starting task 99.0 in stage 0.0 (TID 99) (hopsworks0.logicalclocks.com, executor 1, partition 99, PROCESS_LOCAL, 4329 bytes) taskResourceAssignments Map()\n",
      "2022-04-12 13:24:01,044 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 98.0 in stage 0.0 (TID 98) in 35 ms on hopsworks0.logicalclocks.com (executor 1) (98/100)\n",
      "2022-04-12 13:24:01,097 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 99.0 in stage 0.0 (TID 99) in 56 ms on hopsworks0.logicalclocks.com (executor 1) (99/100)\n",
      "2022-04-12 13:24:01,139 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Resource profile 0 doesn't exist, adding it\n",
      "2022-04-12 13:24:01,140 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnAllocator: Driver requested a total number of 1 executor(s) for resource profile id: 0.\n",
      "2022-04-12 13:24:01,989 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerInfo: Added broadcast_0_piece0 in memory on hopsworks0.logicalclocks.com:37058 (size: 1736.0 B, free: 2004.6 MiB)\n",
      "2022-04-12 13:24:03,391 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(driver)\n",
      "2022-04-12 13:24:03,392 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=driver, job=application_1649425494064_0008\n",
      "2022-04-12 13:24:03,792 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TaskSetManager: Finished task 89.0 in stage 0.0 (TID 89) in 3281 ms on hopsworks0.logicalclocks.com (executor 2) (100/100)\n",
      "2022-04-12 13:24:03,793 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterScheduler: Removed TaskSet 0.0, whose tasks have all completed, from pool \n",
      "2022-04-12 13:24:03,795 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: ResultStage 0 (reduce at SparkPi.scala:38) finished in 11.956 s\n",
      "2022-04-12 13:24:03,801 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Job 0 is finished. Cancelling potential speculative or zombie tasks for this job\n",
      "2022-04-12 13:24:03,802 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterScheduler: Killing all running tasks in stage 0: Stage finished\n",
      "2022-04-12 13:24:03,804 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DAGScheduler: Job 0 finished: reduce at SparkPi.scala:38, took 12.072676 s\n",
      "2022-04-12 13:24:03,849 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 AbstractConnector: Stopped Spark@68ee8eb8{HTTP/1.1, (http/1.1)}{0.0.0.0:0}\n",
      "2022-04-12 13:24:03,857 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkUI: Stopped Spark web UI at http://hopsworks0.logicalclocks.com:41268\n",
      "2022-04-12 13:24:03,860 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(driver)\n",
      "2022-04-12 13:24:03,860 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=driver, job=application_1649425494064_0008\n",
      "2022-04-12 13:24:03,910 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnClusterSchedulerBackend: Shutting down all executors\n",
      "2022-04-12 13:24:03,912 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnSchedulerBackend$YarnDriverEndpoint: Asking each executor to shut down\n",
      "2022-04-12 13:24:04,147 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!\n",
      "2022-04-12 13:24:04,165 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: MemoryStore cleared\n",
      "2022-04-12 13:24:04,166 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: BlockManager stopped\n",
      "2022-04-12 13:24:04,174 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: BlockManagerMaster stopped\n",
      "2022-04-12 13:24:04,179 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!\n",
      "2022-04-12 13:24:04,194 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SparkContext: Successfully stopped SparkContext\n",
      "2022-04-12 13:24:04,196 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Final app status: SUCCEEDED, exitCode: 0\n",
      "2022-04-12 13:24:04,199 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(driver)\n",
      "2022-04-12 13:24:04,199 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=driver, job=application_1649425494064_0008\n",
      "2022-04-12 13:24:04,213 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Unregistering ApplicationMaster with SUCCEEDED\n",
      "2022-04-12 13:24:04,226 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 AMRMClientImpl: Waiting for application to be successfully unregistered.\n",
      "2022-04-12 13:24:04,332 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ApplicationMaster: Deleting staging directory /Projects/demo_ml_meb10000/Resources/.sparkjobstaging-application_1649425494064_0008\n",
      "2022-04-12 13:24:04,382 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ShutdownHookManager: Shutdown hook called\n",
      "2022-04-12 13:24:04,384 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ShutdownHookManager: Deleting directory /srv/hops/hopsdata/tmp/nm-local-dir/usercache/demo_ml_meb10000__meb10000/appcache/application_1649425494064_0008/spark-fd231910-c3a7-4571-aaac-551670530469\n",
      "\n",
      "Log Type: stderr.txt\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000003 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: prelaunch.err\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000003 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: stderr\n",
      "Log Length: 10062\n",
      "Log Contents: \n",
      "2022-04-12 13:23:55,618 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 CoarseGrainedExecutorBackend: Started daemon with process name: 17@hopsworks0.logicalclocks.com\n",
      "2022-04-12 13:23:55,628 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for TERM\n",
      "2022-04-12 13:23:55,629 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for HUP\n",
      "2022-04-12 13:23:55,630 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for INT\n",
      "2022-04-12 13:23:57,229 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:57,230 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:57,235 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls groups to: \n",
      "2022-04-12 13:23:57,238 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls groups to: \n",
      "2022-04-12 13:23:57,239 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with view permissions: Set(); users  with modify permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with modify permissions: Set()\n",
      "2022-04-12 13:23:58,522 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:35603 after 227 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:23:58,780 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:58,780 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:58,780 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls groups to: \n",
      "2022-04-12 13:23:58,780 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls groups to: \n",
      "2022-04-12 13:23:58,780 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with view permissions: Set(); users  with modify permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with modify permissions: Set()\n",
      "2022-04-12 13:23:58,920 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:35603 after 26 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:23:59,084 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DiskBlockManager: Created local directory at /srv/hops/hopsdata/tmp/nm-local-dir/usercache/demo_ml_meb10000__meb10000/appcache/application_1649425494064_0008/blockmgr-829932c1-69db-4671-9af1-8ea927fa4d4d\n",
      "2022-04-12 13:23:59,163 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: MemoryStore started with capacity 2004.6 MiB\n",
      "2022-04-12 13:23:59,653 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Initializing Prometheus Sink...\n",
      "2022-04-12 13:23:59,653 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics polling period -> 10 SECONDS\n",
      "2022-04-12 13:23:59,654 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics timestamp enabled -> false\n",
      "2022-04-12 13:23:59,655 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address -> pushgateway.prometheus.service.consul:9095\n",
      "2022-04-12 13:23:59,655 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address-protocol -> http\n",
      "2022-04-12 13:23:59,656 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-capture-regex -> \n",
      "2022-04-12 13:23:59,656 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-replacement -> \n",
      "2022-04-12 13:23:59,658 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: labels -> Map()\n",
      "2022-04-12 13:23:59,659 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: jmx-collector-config -> /opt/spark/conf/jmx_collector.yaml\n",
      "2022-04-12 13:23:59,798 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@hopsworks0.logicalclocks.com:35603\n",
      "2022-04-12 13:23:59,861 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: ==============================================================\n",
      "2022-04-12 13:23:59,861 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: No custom resources configured for spark.executor.\n",
      "2022-04-12 13:23:59,862 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: ==============================================================\n",
      "2022-04-12 13:24:00,157 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Successfully registered with driver\n",
      "2022-04-12 13:24:00,184 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Starting executor ID 2 on host hopsworks0.logicalclocks.com\n",
      "2022-04-12 13:24:00,370 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 37058.\n",
      "2022-04-12 13:24:00,370 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 NettyBlockTransferService: Server created on hopsworks0.logicalclocks.com:37058\n",
      "2022-04-12 13:24:00,373 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy\n",
      "2022-04-12 13:24:00,387 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: Registering BlockManager BlockManagerId(2, hopsworks0.logicalclocks.com, 37058, None)\n",
      "2022-04-12 13:24:00,412 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: Registered BlockManager BlockManagerId(2, hopsworks0.logicalclocks.com, 37058, None)\n",
      "2022-04-12 13:24:00,413 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: external shuffle service port = 7337\n",
      "2022-04-12 13:24:00,414 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Registering executor with local external shuffle service.\n",
      "2022-04-12 13:24:00,427 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:7337 after 8 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:24:00,444 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Initialized BlockManager: BlockManagerId(2, hopsworks0.logicalclocks.com, 37058, None)\n",
      "2022-04-12 13:24:00,533 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 89\n",
      "2022-04-12 13:24:00,549 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 89.0 in stage 0.0 (TID 89)\n",
      "2022-04-12 13:24:01,811 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TorrentBroadcast: Started reading broadcast variable 0 with 1 pieces (estimated total size 4.0 MiB)\n",
      "2022-04-12 13:24:01,879 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:40427 after 7 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:24:01,972 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1736.0 B, free 2004.6 MiB)\n",
      "2022-04-12 13:24:02,008 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TorrentBroadcast: Reading broadcast variable 0 took 196 ms\n",
      "2022-04-12 13:24:02,108 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: Block broadcast_0 stored as values in memory (estimated size 3.1 KiB, free 2004.6 MiB)\n",
      "2022-04-12 13:24:03,785 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 89.0 in stage 0.0 (TID 89). 885 bytes result sent to driver\n",
      "2022-04-12 13:24:04,418 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Driver commanded a shutdown\n",
      "2022-04-12 13:24:04,420 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Driver from hopsworks0.logicalclocks.com:35603 disconnected during shutdown\n",
      "2022-04-12 13:24:04,420 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Driver from hopsworks0.logicalclocks.com:35603 disconnected during shutdown\n",
      "2022-04-12 13:24:04,435 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(2)\n",
      "2022-04-12 13:24:04,439 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=executor, job=application_1649425494064_0008\n",
      "2022-04-12 13:24:04,576 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: MemoryStore cleared\n",
      "2022-04-12 13:24:04,576 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: BlockManager stopped\n",
      "2022-04-12 13:24:04,588 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ShutdownHookManager: Shutdown hook called\n",
      "\n",
      "Log Type: stderr.txt\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000002 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: prelaunch.err\n",
      "Log Length: 0\n",
      "Container: container_1649425494064_0008_01_000002 on hopsworks0.logicalclocks.com_9000_1649769846113\n",
      "============================================================================================== \n",
      "Log Type: stderr\n",
      "Log Length: 54093\n",
      "Log Contents: \n",
      "2022-04-12 13:23:46,435 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 CoarseGrainedExecutorBackend: Started daemon with process name: 17@hopsworks0.logicalclocks.com\n",
      "2022-04-12 13:23:46,448 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for TERM\n",
      "2022-04-12 13:23:46,449 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for HUP\n",
      "2022-04-12 13:23:46,450 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SignalUtils: Registering signal handler for INT\n",
      "2022-04-12 13:23:48,001 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:48,005 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:48,006 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls groups to: \n",
      "2022-04-12 13:23:48,007 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls groups to: \n",
      "2022-04-12 13:23:48,008 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with view permissions: Set(); users  with modify permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with modify permissions: Set()\n",
      "2022-04-12 13:23:49,213 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:35603 after 125 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:23:49,805 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:49,806 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls to: yarnapp,demo_ml_meb10000__meb10000\n",
      "2022-04-12 13:23:49,806 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing view acls groups to: \n",
      "2022-04-12 13:23:49,806 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: Changing modify acls groups to: \n",
      "2022-04-12 13:23:49,807 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with view permissions: Set(); users  with modify permissions: Set(yarnapp, demo_ml_meb10000__meb10000); groups with modify permissions: Set()\n",
      "2022-04-12 13:23:49,910 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:35603 after 9 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:23:50,016 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 DiskBlockManager: Created local directory at /srv/hops/hopsdata/tmp/nm-local-dir/usercache/demo_ml_meb10000__meb10000/appcache/application_1649425494064_0008/blockmgr-88011c65-6ff6-4789-89fe-52915fa50e18\n",
      "2022-04-12 13:23:50,101 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: MemoryStore started with capacity 2004.6 MiB\n",
      "2022-04-12 13:23:50,756 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Initializing Prometheus Sink...\n",
      "2022-04-12 13:23:50,756 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics polling period -> 10 SECONDS\n",
      "2022-04-12 13:23:50,757 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: Metrics timestamp enabled -> false\n",
      "2022-04-12 13:23:50,757 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address -> pushgateway.prometheus.service.consul:9095\n",
      "2022-04-12 13:23:50,758 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: pushgateway-address-protocol -> http\n",
      "2022-04-12 13:23:50,759 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-capture-regex -> \n",
      "2022-04-12 13:23:50,759 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metrics-name-replacement -> \n",
      "2022-04-12 13:23:50,761 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: labels -> Map()\n",
      "2022-04-12 13:23:50,761 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: jmx-collector-config -> /opt/spark/conf/jmx_collector.yaml\n",
      "2022-04-12 13:23:50,885 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@hopsworks0.logicalclocks.com:35603\n",
      "2022-04-12 13:23:50,906 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: ==============================================================\n",
      "2022-04-12 13:23:50,906 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: No custom resources configured for spark.executor.\n",
      "2022-04-12 13:23:50,907 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ResourceUtils: ==============================================================\n",
      "2022-04-12 13:23:51,132 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Successfully registered with driver\n",
      "2022-04-12 13:23:51,140 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Starting executor ID 1 on host hopsworks0.logicalclocks.com\n",
      "2022-04-12 13:23:51,548 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 40427.\n",
      "2022-04-12 13:23:51,549 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 NettyBlockTransferService: Server created on hopsworks0.logicalclocks.com:40427\n",
      "2022-04-12 13:23:51,551 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy\n",
      "2022-04-12 13:23:51,564 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: Registering BlockManager BlockManagerId(1, hopsworks0.logicalclocks.com, 40427, None)\n",
      "2022-04-12 13:23:51,603 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManagerMaster: Registered BlockManager BlockManagerId(1, hopsworks0.logicalclocks.com, 40427, None)\n",
      "2022-04-12 13:23:51,604 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: external shuffle service port = 7337\n",
      "2022-04-12 13:23:51,605 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Registering executor with local external shuffle service.\n",
      "2022-04-12 13:23:51,618 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:7337 after 9 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:23:51,632 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: Initialized BlockManager: BlockManagerId(1, hopsworks0.logicalclocks.com, 40427, None)\n",
      "2022-04-12 13:23:53,021 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 0\n",
      "2022-04-12 13:23:53,057 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 0.0 in stage 0.0 (TID 0)\n",
      "2022-04-12 13:23:54,452 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TorrentBroadcast: Started reading broadcast variable 0 with 1 pieces (estimated total size 4.0 MiB)\n",
      "2022-04-12 13:23:54,511 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TransportClientFactory: Successfully created connection to hopsworks0.logicalclocks.com/10.0.2.15:41374 after 3 ms (0 ms spent in bootstraps)\n",
      "2022-04-12 13:23:54,588 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1736.0 B, free 2004.6 MiB)\n",
      "2022-04-12 13:23:54,609 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 TorrentBroadcast: Reading broadcast variable 0 took 156 ms\n",
      "2022-04-12 13:23:54,684 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: Block broadcast_0 stored as values in memory (estimated size 3.1 KiB, free 2004.6 MiB)\n",
      "2022-04-12 13:23:56,215 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 0.0 in stage 0.0 (TID 0). 842 bytes result sent to driver\n",
      "2022-04-12 13:23:56,250 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 1\n",
      "2022-04-12 13:23:56,250 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 1.0 in stage 0.0 (TID 1)\n",
      "2022-04-12 13:23:56,305 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 1.0 in stage 0.0 (TID 1). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,324 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 2\n",
      "2022-04-12 13:23:56,325 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 2.0 in stage 0.0 (TID 2)\n",
      "2022-04-12 13:23:56,384 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 2.0 in stage 0.0 (TID 2). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,414 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 3\n",
      "2022-04-12 13:23:56,415 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 3.0 in stage 0.0 (TID 3)\n",
      "2022-04-12 13:23:56,434 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 3.0 in stage 0.0 (TID 3). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,449 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 4\n",
      "2022-04-12 13:23:56,450 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 4.0 in stage 0.0 (TID 4)\n",
      "2022-04-12 13:23:56,471 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 4.0 in stage 0.0 (TID 4). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,485 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 5\n",
      "2022-04-12 13:23:56,485 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 5.0 in stage 0.0 (TID 5)\n",
      "2022-04-12 13:23:56,517 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 5.0 in stage 0.0 (TID 5). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,529 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 6\n",
      "2022-04-12 13:23:56,529 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 6.0 in stage 0.0 (TID 6)\n",
      "2022-04-12 13:23:56,548 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 6.0 in stage 0.0 (TID 6). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,555 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 7\n",
      "2022-04-12 13:23:56,556 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 7.0 in stage 0.0 (TID 7)\n",
      "2022-04-12 13:23:56,585 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 7.0 in stage 0.0 (TID 7). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,607 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 8\n",
      "2022-04-12 13:23:56,608 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 8.0 in stage 0.0 (TID 8)\n",
      "2022-04-12 13:23:56,630 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 8.0 in stage 0.0 (TID 8). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,653 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 9\n",
      "2022-04-12 13:23:56,654 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 9.0 in stage 0.0 (TID 9)\n",
      "2022-04-12 13:23:56,686 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 9.0 in stage 0.0 (TID 9). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,712 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 10\n",
      "2022-04-12 13:23:56,713 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 10.0 in stage 0.0 (TID 10)\n",
      "2022-04-12 13:23:56,732 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 10.0 in stage 0.0 (TID 10). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,740 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 11\n",
      "2022-04-12 13:23:56,741 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 11.0 in stage 0.0 (TID 11)\n",
      "2022-04-12 13:23:56,767 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 11.0 in stage 0.0 (TID 11). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,775 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 12\n",
      "2022-04-12 13:23:56,775 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 12.0 in stage 0.0 (TID 12)\n",
      "2022-04-12 13:23:56,795 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 12.0 in stage 0.0 (TID 12). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,805 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 13\n",
      "2022-04-12 13:23:56,806 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 13.0 in stage 0.0 (TID 13)\n",
      "2022-04-12 13:23:56,825 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 13.0 in stage 0.0 (TID 13). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,835 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 14\n",
      "2022-04-12 13:23:56,835 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 14.0 in stage 0.0 (TID 14)\n",
      "2022-04-12 13:23:56,852 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 14.0 in stage 0.0 (TID 14). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,864 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 15\n",
      "2022-04-12 13:23:56,865 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 15.0 in stage 0.0 (TID 15)\n",
      "2022-04-12 13:23:56,885 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 15.0 in stage 0.0 (TID 15). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,897 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 16\n",
      "2022-04-12 13:23:56,897 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 16.0 in stage 0.0 (TID 16)\n",
      "2022-04-12 13:23:56,914 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 16.0 in stage 0.0 (TID 16). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,922 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 17\n",
      "2022-04-12 13:23:56,923 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 17.0 in stage 0.0 (TID 17)\n",
      "2022-04-12 13:23:56,939 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 17.0 in stage 0.0 (TID 17). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,948 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 18\n",
      "2022-04-12 13:23:56,948 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 18.0 in stage 0.0 (TID 18)\n",
      "2022-04-12 13:23:56,966 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 18.0 in stage 0.0 (TID 18). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:56,976 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 19\n",
      "2022-04-12 13:23:56,978 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 19.0 in stage 0.0 (TID 19)\n",
      "2022-04-12 13:23:57,009 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 19.0 in stage 0.0 (TID 19). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,022 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 20\n",
      "2022-04-12 13:23:57,023 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 20.0 in stage 0.0 (TID 20)\n",
      "2022-04-12 13:23:57,041 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 20.0 in stage 0.0 (TID 20). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,062 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 21\n",
      "2022-04-12 13:23:57,062 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 21.0 in stage 0.0 (TID 21)\n",
      "2022-04-12 13:23:57,088 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 21.0 in stage 0.0 (TID 21). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,103 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 22\n",
      "2022-04-12 13:23:57,103 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 22.0 in stage 0.0 (TID 22)\n",
      "2022-04-12 13:23:57,129 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 22.0 in stage 0.0 (TID 22). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,150 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 23\n",
      "2022-04-12 13:23:57,151 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 23.0 in stage 0.0 (TID 23)\n",
      "2022-04-12 13:23:57,176 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 23.0 in stage 0.0 (TID 23). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,186 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 24\n",
      "2022-04-12 13:23:57,188 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 24.0 in stage 0.0 (TID 24)\n",
      "2022-04-12 13:23:57,211 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 24.0 in stage 0.0 (TID 24). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,219 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 25\n",
      "2022-04-12 13:23:57,219 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 25.0 in stage 0.0 (TID 25)\n",
      "2022-04-12 13:23:57,236 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 25.0 in stage 0.0 (TID 25). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,257 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 26\n",
      "2022-04-12 13:23:57,258 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 26.0 in stage 0.0 (TID 26)\n",
      "2022-04-12 13:23:57,300 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 26.0 in stage 0.0 (TID 26). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,319 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 27\n",
      "2022-04-12 13:23:57,320 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 27.0 in stage 0.0 (TID 27)\n",
      "2022-04-12 13:23:57,361 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 27.0 in stage 0.0 (TID 27). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,367 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 28\n",
      "2022-04-12 13:23:57,367 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 28.0 in stage 0.0 (TID 28)\n",
      "2022-04-12 13:23:57,404 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 28.0 in stage 0.0 (TID 28). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,427 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 29\n",
      "2022-04-12 13:23:57,428 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 29.0 in stage 0.0 (TID 29)\n",
      "2022-04-12 13:23:57,448 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 29.0 in stage 0.0 (TID 29). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,462 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 30\n",
      "2022-04-12 13:23:57,463 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 30.0 in stage 0.0 (TID 30)\n",
      "2022-04-12 13:23:57,482 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 30.0 in stage 0.0 (TID 30). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,498 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 31\n",
      "2022-04-12 13:23:57,499 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 31.0 in stage 0.0 (TID 31)\n",
      "2022-04-12 13:23:57,517 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 31.0 in stage 0.0 (TID 31). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,537 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 32\n",
      "2022-04-12 13:23:57,538 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 32.0 in stage 0.0 (TID 32)\n",
      "2022-04-12 13:23:57,555 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 32.0 in stage 0.0 (TID 32). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,565 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 33\n",
      "2022-04-12 13:23:57,565 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 33.0 in stage 0.0 (TID 33)\n",
      "2022-04-12 13:23:57,601 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 33.0 in stage 0.0 (TID 33). 842 bytes result sent to driver\n",
      "2022-04-12 13:23:57,609 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 34\n",
      "2022-04-12 13:23:57,609 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 34.0 in stage 0.0 (TID 34)\n",
      "2022-04-12 13:23:57,628 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 34.0 in stage 0.0 (TID 34). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,643 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 35\n",
      "2022-04-12 13:23:57,644 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 35.0 in stage 0.0 (TID 35)\n",
      "2022-04-12 13:23:57,664 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 35.0 in stage 0.0 (TID 35). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,674 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 36\n",
      "2022-04-12 13:23:57,676 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 36.0 in stage 0.0 (TID 36)\n",
      "2022-04-12 13:23:57,727 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 36.0 in stage 0.0 (TID 36). 842 bytes result sent to driver\n",
      "2022-04-12 13:23:57,740 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 37\n",
      "2022-04-12 13:23:57,741 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 37.0 in stage 0.0 (TID 37)\n",
      "2022-04-12 13:23:57,798 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 37.0 in stage 0.0 (TID 37). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,822 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 38\n",
      "2022-04-12 13:23:57,822 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 38.0 in stage 0.0 (TID 38)\n",
      "2022-04-12 13:23:57,844 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 38.0 in stage 0.0 (TID 38). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,866 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 39\n",
      "2022-04-12 13:23:57,866 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 39.0 in stage 0.0 (TID 39)\n",
      "2022-04-12 13:23:57,881 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 39.0 in stage 0.0 (TID 39). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,892 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 40\n",
      "2022-04-12 13:23:57,893 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 40.0 in stage 0.0 (TID 40)\n",
      "2022-04-12 13:23:57,937 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 40.0 in stage 0.0 (TID 40). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:57,958 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 41\n",
      "2022-04-12 13:23:57,959 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 41.0 in stage 0.0 (TID 41)\n",
      "2022-04-12 13:23:57,981 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 41.0 in stage 0.0 (TID 41). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,011 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 42\n",
      "2022-04-12 13:23:58,011 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 42.0 in stage 0.0 (TID 42)\n",
      "2022-04-12 13:23:58,068 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 42.0 in stage 0.0 (TID 42). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,083 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 43\n",
      "2022-04-12 13:23:58,083 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 43.0 in stage 0.0 (TID 43)\n",
      "2022-04-12 13:23:58,137 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 43.0 in stage 0.0 (TID 43). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,156 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 44\n",
      "2022-04-12 13:23:58,157 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 44.0 in stage 0.0 (TID 44)\n",
      "2022-04-12 13:23:58,187 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 44.0 in stage 0.0 (TID 44). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,199 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 45\n",
      "2022-04-12 13:23:58,201 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 45.0 in stage 0.0 (TID 45)\n",
      "2022-04-12 13:23:58,241 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 45.0 in stage 0.0 (TID 45). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,252 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 46\n",
      "2022-04-12 13:23:58,253 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 46.0 in stage 0.0 (TID 46)\n",
      "2022-04-12 13:23:58,295 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 46.0 in stage 0.0 (TID 46). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,355 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 47\n",
      "2022-04-12 13:23:58,356 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 47.0 in stage 0.0 (TID 47)\n",
      "2022-04-12 13:23:58,426 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 47.0 in stage 0.0 (TID 47). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,446 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 48\n",
      "2022-04-12 13:23:58,447 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 48.0 in stage 0.0 (TID 48)\n",
      "2022-04-12 13:23:58,473 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 48.0 in stage 0.0 (TID 48). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,493 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 49\n",
      "2022-04-12 13:23:58,495 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 49.0 in stage 0.0 (TID 49)\n",
      "2022-04-12 13:23:58,507 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 49.0 in stage 0.0 (TID 49). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,517 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 50\n",
      "2022-04-12 13:23:58,517 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 50.0 in stage 0.0 (TID 50)\n",
      "2022-04-12 13:23:58,881 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 50.0 in stage 0.0 (TID 50). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,896 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 51\n",
      "2022-04-12 13:23:58,896 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 51.0 in stage 0.0 (TID 51)\n",
      "2022-04-12 13:23:58,915 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 51.0 in stage 0.0 (TID 51). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,937 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 52\n",
      "2022-04-12 13:23:58,938 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 52.0 in stage 0.0 (TID 52)\n",
      "2022-04-12 13:23:58,958 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 52.0 in stage 0.0 (TID 52). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:58,981 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 53\n",
      "2022-04-12 13:23:58,982 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 53.0 in stage 0.0 (TID 53)\n",
      "2022-04-12 13:23:59,002 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 53.0 in stage 0.0 (TID 53). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,012 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 54\n",
      "2022-04-12 13:23:59,013 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 54.0 in stage 0.0 (TID 54)\n",
      "2022-04-12 13:23:59,028 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 54.0 in stage 0.0 (TID 54). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,046 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 55\n",
      "2022-04-12 13:23:59,047 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 55.0 in stage 0.0 (TID 55)\n",
      "2022-04-12 13:23:59,063 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 55.0 in stage 0.0 (TID 55). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,070 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 56\n",
      "2022-04-12 13:23:59,070 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 56.0 in stage 0.0 (TID 56)\n",
      "2022-04-12 13:23:59,085 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 56.0 in stage 0.0 (TID 56). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,092 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 57\n",
      "2022-04-12 13:23:59,095 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 57.0 in stage 0.0 (TID 57)\n",
      "2022-04-12 13:23:59,109 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 57.0 in stage 0.0 (TID 57). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,122 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 58\n",
      "2022-04-12 13:23:59,123 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 58.0 in stage 0.0 (TID 58)\n",
      "2022-04-12 13:23:59,139 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 58.0 in stage 0.0 (TID 58). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,147 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 59\n",
      "2022-04-12 13:23:59,150 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 59.0 in stage 0.0 (TID 59)\n",
      "2022-04-12 13:23:59,176 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 59.0 in stage 0.0 (TID 59). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,188 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 60\n",
      "2022-04-12 13:23:59,195 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 60.0 in stage 0.0 (TID 60)\n",
      "2022-04-12 13:23:59,228 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 60.0 in stage 0.0 (TID 60). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,245 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 61\n",
      "2022-04-12 13:23:59,246 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 61.0 in stage 0.0 (TID 61)\n",
      "2022-04-12 13:23:59,264 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 61.0 in stage 0.0 (TID 61). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,278 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 62\n",
      "2022-04-12 13:23:59,278 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 62.0 in stage 0.0 (TID 62)\n",
      "2022-04-12 13:23:59,292 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 62.0 in stage 0.0 (TID 62). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,313 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 63\n",
      "2022-04-12 13:23:59,314 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 63.0 in stage 0.0 (TID 63)\n",
      "2022-04-12 13:23:59,333 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 63.0 in stage 0.0 (TID 63). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,348 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 64\n",
      "2022-04-12 13:23:59,348 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 64.0 in stage 0.0 (TID 64)\n",
      "2022-04-12 13:23:59,372 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 64.0 in stage 0.0 (TID 64). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,392 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 65\n",
      "2022-04-12 13:23:59,392 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 65.0 in stage 0.0 (TID 65)\n",
      "2022-04-12 13:23:59,410 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 65.0 in stage 0.0 (TID 65). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,447 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 66\n",
      "2022-04-12 13:23:59,447 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 66.0 in stage 0.0 (TID 66)\n",
      "2022-04-12 13:23:59,467 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 66.0 in stage 0.0 (TID 66). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,483 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 67\n",
      "2022-04-12 13:23:59,483 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 67.0 in stage 0.0 (TID 67)\n",
      "2022-04-12 13:23:59,799 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 67.0 in stage 0.0 (TID 67). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,811 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 68\n",
      "2022-04-12 13:23:59,812 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 68.0 in stage 0.0 (TID 68)\n",
      "2022-04-12 13:23:59,826 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 68.0 in stage 0.0 (TID 68). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,854 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 69\n",
      "2022-04-12 13:23:59,855 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 69.0 in stage 0.0 (TID 69)\n",
      "2022-04-12 13:23:59,871 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 69.0 in stage 0.0 (TID 69). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,884 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 70\n",
      "2022-04-12 13:23:59,885 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 70.0 in stage 0.0 (TID 70)\n",
      "2022-04-12 13:23:59,912 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 70.0 in stage 0.0 (TID 70). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,927 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 71\n",
      "2022-04-12 13:23:59,928 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 71.0 in stage 0.0 (TID 71)\n",
      "2022-04-12 13:23:59,942 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 71.0 in stage 0.0 (TID 71). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,951 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 72\n",
      "2022-04-12 13:23:59,954 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 72.0 in stage 0.0 (TID 72)\n",
      "2022-04-12 13:23:59,970 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 72.0 in stage 0.0 (TID 72). 799 bytes result sent to driver\n",
      "2022-04-12 13:23:59,981 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 73\n",
      "2022-04-12 13:23:59,982 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 73.0 in stage 0.0 (TID 73)\n",
      "2022-04-12 13:24:00,008 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 73.0 in stage 0.0 (TID 73). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,040 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 74\n",
      "2022-04-12 13:24:00,041 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 74.0 in stage 0.0 (TID 74)\n",
      "2022-04-12 13:24:00,058 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 74.0 in stage 0.0 (TID 74). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,064 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 75\n",
      "2022-04-12 13:24:00,064 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 75.0 in stage 0.0 (TID 75)\n",
      "2022-04-12 13:24:00,079 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 75.0 in stage 0.0 (TID 75). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,094 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 76\n",
      "2022-04-12 13:24:00,094 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 76.0 in stage 0.0 (TID 76)\n",
      "2022-04-12 13:24:00,108 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 76.0 in stage 0.0 (TID 76). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,122 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 77\n",
      "2022-04-12 13:24:00,122 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 77.0 in stage 0.0 (TID 77)\n",
      "2022-04-12 13:24:00,143 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 77.0 in stage 0.0 (TID 77). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,157 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 78\n",
      "2022-04-12 13:24:00,157 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 78.0 in stage 0.0 (TID 78)\n",
      "2022-04-12 13:24:00,171 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 78.0 in stage 0.0 (TID 78). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,179 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 79\n",
      "2022-04-12 13:24:00,179 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 79.0 in stage 0.0 (TID 79)\n",
      "2022-04-12 13:24:00,194 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 79.0 in stage 0.0 (TID 79). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,201 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 80\n",
      "2022-04-12 13:24:00,202 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 80.0 in stage 0.0 (TID 80)\n",
      "2022-04-12 13:24:00,217 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 80.0 in stage 0.0 (TID 80). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,223 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 81\n",
      "2022-04-12 13:24:00,224 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 81.0 in stage 0.0 (TID 81)\n",
      "2022-04-12 13:24:00,238 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 81.0 in stage 0.0 (TID 81). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,245 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 82\n",
      "2022-04-12 13:24:00,245 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 82.0 in stage 0.0 (TID 82)\n",
      "2022-04-12 13:24:00,267 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 82.0 in stage 0.0 (TID 82). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,273 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 83\n",
      "2022-04-12 13:24:00,274 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 83.0 in stage 0.0 (TID 83)\n",
      "2022-04-12 13:24:00,292 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 83.0 in stage 0.0 (TID 83). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,298 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 84\n",
      "2022-04-12 13:24:00,300 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 84.0 in stage 0.0 (TID 84)\n",
      "2022-04-12 13:24:00,314 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 84.0 in stage 0.0 (TID 84). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,325 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 85\n",
      "2022-04-12 13:24:00,325 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 85.0 in stage 0.0 (TID 85)\n",
      "2022-04-12 13:24:00,340 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 85.0 in stage 0.0 (TID 85). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,349 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 86\n",
      "2022-04-12 13:24:00,350 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 86.0 in stage 0.0 (TID 86)\n",
      "2022-04-12 13:24:00,369 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 86.0 in stage 0.0 (TID 86). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,376 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 87\n",
      "2022-04-12 13:24:00,376 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 87.0 in stage 0.0 (TID 87)\n",
      "2022-04-12 13:24:00,391 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 87.0 in stage 0.0 (TID 87). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,408 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 88\n",
      "2022-04-12 13:24:00,718 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 88.0 in stage 0.0 (TID 88)\n",
      "2022-04-12 13:24:00,777 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 88.0 in stage 0.0 (TID 88). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,792 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 90\n",
      "2022-04-12 13:24:00,792 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 90.0 in stage 0.0 (TID 90)\n",
      "2022-04-12 13:24:00,810 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 90.0 in stage 0.0 (TID 90). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,824 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(1)\n",
      "2022-04-12 13:24:00,826 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 91\n",
      "2022-04-12 13:24:00,827 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=executor, job=application_1649425494064_0008\n",
      "2022-04-12 13:24:00,827 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 91.0 in stage 0.0 (TID 91)\n",
      "2022-04-12 13:24:00,842 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 91.0 in stage 0.0 (TID 91). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,852 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 92\n",
      "2022-04-12 13:24:00,852 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 92.0 in stage 0.0 (TID 92)\n",
      "2022-04-12 13:24:00,865 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 92.0 in stage 0.0 (TID 92). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,875 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 93\n",
      "2022-04-12 13:24:00,875 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 93.0 in stage 0.0 (TID 93)\n",
      "2022-04-12 13:24:00,907 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 93.0 in stage 0.0 (TID 93). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,913 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 94\n",
      "2022-04-12 13:24:00,913 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 94.0 in stage 0.0 (TID 94)\n",
      "2022-04-12 13:24:00,927 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 94.0 in stage 0.0 (TID 94). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,939 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 95\n",
      "2022-04-12 13:24:00,941 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 95.0 in stage 0.0 (TID 95)\n",
      "2022-04-12 13:24:00,960 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 95.0 in stage 0.0 (TID 95). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,966 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 96\n",
      "2022-04-12 13:24:00,967 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 96.0 in stage 0.0 (TID 96)\n",
      "2022-04-12 13:24:00,982 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 96.0 in stage 0.0 (TID 96). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:00,988 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 97\n",
      "2022-04-12 13:24:00,989 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 97.0 in stage 0.0 (TID 97)\n",
      "2022-04-12 13:24:01,005 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 97.0 in stage 0.0 (TID 97). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:01,013 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 98\n",
      "2022-04-12 13:24:01,013 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 98.0 in stage 0.0 (TID 98)\n",
      "2022-04-12 13:24:01,033 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 98.0 in stage 0.0 (TID 98). 842 bytes result sent to driver\n",
      "2022-04-12 13:24:01,048 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Got assigned task 99\n",
      "2022-04-12 13:24:01,049 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Running task 99.0 in stage 0.0 (TID 99)\n",
      "2022-04-12 13:24:01,076 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 Executor: Finished task 99.0 in stage 0.0 (TID 99). 799 bytes result sent to driver\n",
      "2022-04-12 13:24:03,925 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 YarnCoarseGrainedExecutorBackend: Driver commanded a shutdown\n",
      "2022-04-12 13:24:03,928 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: metricsNamespace=None, sparkAppName=Some(Spark Pi), sparkAppId=Some(application_1649425494064_0008), executorId=Some(1)\n",
      "2022-04-12 13:24:03,928 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 PrometheusSink: role=executor, job=application_1649425494064_0008\n",
      "2022-04-12 13:24:03,984 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 MemoryStore: MemoryStore cleared\n",
      "2022-04-12 13:24:03,985 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 BlockManager: BlockManager stopped\n",
      "2022-04-12 13:24:04,005 INFO demo_ml_meb10000,my_job,43,application_1649425494064_0008 ShutdownHookManager: Shutdown hook called\n",
      "\n",
      "Log Type: stderr.txt\n",
      "Log Length: 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f_err = open(err, \"r\")\n",
    "print(f_err.read()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "670d074a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Execution('SUCCEEDED', 'FINISHED', '2022-04-12T13:23:21Z', '100')]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all the executions for the job\n",
    "my_job.get_executions()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d27bc76",
   "metadata": {},
   "source": [
    "## Update the configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4d8d1f0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'sparkJobConfiguration',\n",
       " 'appName': 'my_job',\n",
       " 'amQueue': 'default',\n",
       " 'amMemory': 2048,\n",
       " 'amVCores': 1,\n",
       " 'jobType': 'SPARK',\n",
       " 'appPath': 'hdfs:///Projects/demo_ml_meb10000/Resources/spark-examples.jar',\n",
       " 'mainClass': 'org.apache.spark.examples.SparkPi',\n",
       " 'spark.executor.instances': 1,\n",
       " 'spark.executor.cores': 1,\n",
       " 'spark.executor.memory': 4096,\n",
       " 'spark.executor.gpus': 0,\n",
       " 'spark.tensorflow.num.ps': 1,\n",
       " 'spark.dynamicAllocation.enabled': True,\n",
       " 'spark.dynamicAllocation.minExecutors': 1,\n",
       " 'spark.dynamicAllocation.maxExecutors': 2,\n",
       " 'spark.dynamicAllocation.initialExecutors': 1,\n",
       " 'spark.blacklist.enabled': False}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_job.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "726f6459",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = my_job.config\n",
    "config['amMemory'] = 2000 # Update the Spark driver memory to 2000MB\n",
    "my_job.config = config\n",
    "\n",
    "my_job = my_job.save() # Save in the backend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a459d0c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'sparkJobConfiguration',\n",
       " 'appName': 'my_job',\n",
       " 'amQueue': 'default',\n",
       " 'amMemory': 2000,\n",
       " 'amVCores': 1,\n",
       " 'jobType': 'SPARK',\n",
       " 'appPath': 'hdfs:///Projects/demo_ml_meb10000/Resources/spark-examples.jar',\n",
       " 'mainClass': 'org.apache.spark.examples.SparkPi',\n",
       " 'spark.executor.instances': 1,\n",
       " 'spark.executor.cores': 1,\n",
       " 'spark.executor.memory': 4096,\n",
       " 'spark.executor.gpus': 0,\n",
       " 'spark.tensorflow.num.ps': 1,\n",
       " 'spark.dynamicAllocation.enabled': True,\n",
       " 'spark.dynamicAllocation.minExecutors': 1,\n",
       " 'spark.dynamicAllocation.maxExecutors': 2,\n",
       " 'spark.dynamicAllocation.initialExecutors': 1,\n",
       " 'spark.blacklist.enabled': False}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_job.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "cde9af58",
   "metadata": {},
   "outputs": [],
   "source": [
    "my_job.delete()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}