From 74adeec189d5b19acd73a2e2f2c6735e001fae51 Mon Sep 17 00:00:00 2001
From: Satendra Kumar <satendrakumar06@gmail.com>
Date: Sun, 17 Feb 2019 17:14:45 +0530
Subject: [PATCH] Update spark and elasticsearch version

---
 build.sbt                                         |  6 +++---
 .../com/techmonad/pipeline/reader/CSVReader.scala | 15 ++++++++-------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/build.sbt b/build.sbt
index a71257b..bb6f36d 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,13 +2,13 @@ name := "spark-data-pipeline"
 
 version := "1.0"
 
-scalaVersion := "2.11.11"
+scalaVersion := "2.11.12"
 
 
 libraryDependencies ++= Seq(
-  "org.apache.spark" %% "spark-core" % "2.3.1",
+  "org.apache.spark" %% "spark-core" % "2.4.0",
   "com.univocity" % "univocity-parsers" % "2.6.4",
-  "org.elasticsearch" %% "elasticsearch-spark-20" % "6.3.0",
+  "org.elasticsearch" %% "elasticsearch-spark-20" % "6.6.0",
   "edu.stanford.nlp" % "stanford-corenlp" % "3.6.0" artifacts(Artifact("stanford-corenlp", "models"), Artifact("stanford-corenlp")),
   "ch.qos.logback" % "logback-classic" % "1.2.3",
   "org.json4s" %% "json4s-native" % "3.5.4",
diff --git a/src/main/scala/com/techmonad/pipeline/reader/CSVReader.scala b/src/main/scala/com/techmonad/pipeline/reader/CSVReader.scala
index fd7983b..bbc5836 100644
--- a/src/main/scala/com/techmonad/pipeline/reader/CSVReader.scala
+++ b/src/main/scala/com/techmonad/pipeline/reader/CSVReader.scala
@@ -10,13 +10,14 @@ object CSVReader extends TryHelper {
 
   def read(url: String, delimiter: Char = ',')(implicit sc: SparkContext): RDD[Record] = {
     val headers = sc.textFile(url).first()
-    sc.textFile(url).mapPartitionsWithIndex {
-      case (index, itr) =>
-        if (index == 0)
-          readFile(itr.drop(1), headers)
-        else
-          readFile(itr, headers)
-    }
+    sc.textFile(url)
+      .mapPartitionsWithIndex {
+        case (index, itr) =>
+          if (index == 0)
+            readFile(itr.drop(1), headers)
+          else
+            readFile(itr, headers)
+      }
   }