Spark Customer Data Analysis Part 1

Create Data and Schema
-————————-

val data = Seq(
(“gowtham”,“sb”,1,“phone”,“flipkart”,50000),
(“nandhu”,“b”,2,“tv”,“amazon”,65000),
(“rahul”,“m”,3,“laptop”,“ebay”,80000),
(“saravana”,“p”,4,“camera”,“flipkart”,48000),
(“nila”,“gn”,5,“watch”,“amazon”,35000))

val col = Seq(“first_name”,“last_name”,“sno”,“product”,“platform”,“price”)
var df = data.toDF(col:_*)
df.show()

Changing DataType of a column
-—————————————-

import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
println(“before”)
df.printSchema()
df = df.withColumn(“price”,col(“price”).cast(DoubleType))
println(“After”)
df.printSchema()

Adding a new column
-————————-
df = df.withColumn(“offer_price”,col(“price”)-500)
df.show()

Updating the value of an existing column ( adding delivery charge)
-———————————————————————————————-

df = df.withColumn(“offer_price”,col(“offer_price”).plus(50))
df.show()

Dropping an unwanted column
-————————————
df = df.drop(col(“price”))
df.show()

Creating a new column from existing columns
-————————————————————
import org.apache.spark.sql.functions._
df = df.withColumn(“full_name”,concat_ws(" “,col(”first_name"),col(“last_name”)))
df.show()

case clause using when() in withcolumn()
-——————————————————-
import org.apache.spark.sql.functions._
df = df.withColumn(“platform_alais”,when(col(“platform”).equalTo(“amazon”),lit(“a”))
.when(col(“platform”).equalTo(“flipkart”),lit(“f”))
.otherwise(lit(“others”)))

df.show()

Renaming a column using withColumnRenamed()
-—————————————————————
import org.apache.spark.sql.functions._
df = df.withColumnRenamed(“platform”,“app_name”)
df.show()