Add files via upload

Titanic Data Analysis using DataFrames.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"code","source":["titanic_data = spark.read.csv('/FileStore/tables/titanic/titanic_train-ac800.csv', header=True, inferSchema=True)"],"metadata":{},"outputs":[],"execution_count":1},{"cell_type":"code","source":["display(titanic_data)"],"metadata":{},"outputs":[],"execution_count":2},{"cell_type":"code","source":["titanic_data.printSchema()"],"metadata":{},"outputs":[],"execution_count":3},{"cell_type":"code","source":["#Finding maximum fare\ntitanic_data.agg({\"Fare\":\"max\"}).collect()"],"metadata":{},"outputs":[],"execution_count":4},{"cell_type":"code","source":["import pyspark.sql.functions as F"],"metadata":{},"outputs":[],"execution_count":5},{"cell_type":"code","source":["#Finding maximum fare - another way\ntitanic_data.agg(F.max(titanic_data.Fare)).collect()"],"metadata":{},"outputs":[],"execution_count":6},{"cell_type":"code","source":["display(titanic_data.groupBy('Pclass').count())"],"metadata":{},"outputs":[],"execution_count":7},{"cell_type":"code","source":["titanic_data.groupBy('Pclass').avg('Age').collect()"],"metadata":{},"outputs":[],"execution_count":8},{"cell_type":"code","source":["#Descending order of age\ndisplay(titanic_data.orderBy(titanic_data.Age.desc()))"],"metadata":{},"outputs":[],"execution_count":9},{"cell_type":"code","source":["#Person with longest name\ntitanic_data"],"metadata":{},"outputs":[],"execution_count":10},{"cell_type":"code","source":["### UDF in dataframes"],"metadata":{},"outputs":[],"execution_count":11},{"cell_type":"code","source":["from pyspark.sql.functions import udf"],"metadata":{},"outputs":[],"execution_count":12},{"cell_type":"code","source":["def getLen(word):\n  return len(word)"],"metadata":{},"outputs":[],"execution_count":13},{"cell_type":"code","source":["from pyspark.sql.types import IntegerType\nlen_udf = udf(getLen, IntegerType())"],"metadata":{},"outputs":[],"execution_count":14},{"cell_type":"code","source":["titanic_data = titanic_data.select(\"*\", len_udf(\"Name\").alias(\"len_name\"))"],"metadata":{},"outputs":[],"execution_count":15},{"cell_type":"code","source":["titanic_data.agg({'len_name':'max'}).collect()"],"metadata":{},"outputs":[],"execution_count":16},{"cell_type":"code","source":["display(titanic_data[titanic_data.len_name == 82])"],"metadata":{},"outputs":[],"execution_count":17},{"cell_type":"code","source":["\n"],"metadata":{},"outputs":[],"execution_count":18},{"cell_type":"code","source":["titanic_data.select(len_udf(\"Name\").alias(\"len_name\")).len_name"],"metadata":{},"outputs":[],"execution_count":19},{"cell_type":"code","source":["##Remove all the rows in which age is missing"],"metadata":{},"outputs":[],"execution_count":20},{"cell_type":"code","source":["help(titanic_data.dropna)"],"metadata":{},"outputs":[],"execution_count":21},{"cell_type":"code","source":["display(titanic_data.dropna(subset=['Age']))"],"metadata":{},"outputs":[],"execution_count":22},{"cell_type":"code","source":[""],"metadata":{},"outputs":[],"execution_count":23}],"metadata":{"name":"Titanic Data Analysis using DataFrames","notebookId":3739718737890552},"nbformat":4,"nbformat_minor":0}