Spark version update

Author: satendrakumar

README.md

@@ -1 +1,20 @@
 # spark-data-pipeline
+
+#### Elasticsearch Setup
+  i) [Download](https://www.elastic.co/downloads/elasticsearch) the Elasticsearch 6.3.0 or latest version  and unzip it.
+
+  ii) Run the following command.
+
+        $ bin/elasticsearch
+        
+        
+
+#### Getting Started:
+
+ Clone and run in local mode:
+
+        $ git clone git@github.com:techmonad/spark-data-pipeline.git
+        $ cd spark-data-pipeline
+        $ sbt run
+                
+

build.sbt

@@ -2,14 +2,15 @@ name := "spark-data-pipeline"
 
 version := "1.0"
 
-scalaVersion := "2.11.8"
+scalaVersion := "2.11.11"
 
 
 libraryDependencies ++= Seq(
-  "org.apache.spark" %% "spark-core" % "2.1.0",
-  "org.elasticsearch" %% "elasticsearch-spark-20" % "5.6.0",
+  "org.apache.spark" %% "spark-core" % "2.3.1",
+  "com.univocity" % "univocity-parsers" % "2.6.4",
+  "org.elasticsearch" %% "elasticsearch-spark-20" % "6.3.0",
   "edu.stanford.nlp" % "stanford-corenlp" % "3.6.0" artifacts(Artifact("stanford-corenlp", "models"), Artifact("stanford-corenlp")),
   "ch.qos.logback" % "logback-classic" % "1.2.3",
-  "org.json4s" %% "json4s-native" % "3.5.0",
+  "org.json4s" %% "json4s-native" % "3.5.4",
   "org.scalatest" %% "scalatest" % "3.0.1"
 )

src/main/scala/com/techmonad/pipeline/DataPipeline.scala

@@ -36,7 +36,7 @@ object DataPipeline {
     }
   }
 
-  private def applySource(source: Source)(implicit sc: SparkContext) = {
+  private def applySource(source: Source)(implicit sc: SparkContext): RDD[Record] = {
     CSVReader.read(source.path)
   }
 
@@ -54,16 +54,14 @@ object DataPipeline {
     transformations match {
       case Nil => rdd
       case head :: tail =>
-        applyTransformation(
-          Transformations.get(head).map { v => rdd.map(v.transform) }.getOrElse(rdd)
-          , tail)
+        applyTransformation(Transformations.get(head).map { v => rdd.map(v.transform) }.getOrElse(rdd), tail)
     }
 
-  private def applySchemaValidation(rdd: RDD[Record], validations: List[String]) = {
+  private def applySchemaValidation(rdd: RDD[Record], validations: List[String]): RDD[Record] = {
     applyValidation(rdd, validations)
   }
 
-  private def applySink(rdd: RDD[Record], sink: Sink) =
+  private def applySink(rdd: RDD[Record], sink: Sink): ESPersistenceRDD =
     sink.`type` match {
       case "ES" => new ESPersistenceRDD(rdd)
     }