Skip to content
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
Cannot retrieve contributors at this time
package net.janvsmachine.sparksessions
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Dataset, SparkSession}
* Implements sessionization using `groupBy` on an RDD.
object GroupBySessions extends Sessions with Spark {
def sessionize(clicks: Dataset[Click], maxSessionDuration: Long)(implicit spark: SparkSession): Dataset[Session] = {
import spark.implicits._
// Convert a sequence of clicks into sessions.
// Assumes given clicks all belong to same group that we want to create sessions for, e.g. user.
def sessionizeClicks(clicks: Iterable[Click]): Seq[Session] = {
def mergeClickWithSessions(sessions: Seq[Session], click: Click): Seq[Session] =
if (sessions.nonEmpty && click.timestamp <= sessions.head.endTime + maxSessionDuration) {
val previousSession = sessions.head
val updatedSession = previousSession.copy(endTime = click.timestamp, count = previousSession.count + 1)
updatedSession +: sessions.tail
Session(click.userId, click.timestamp, click.timestamp, count = 1) +: sessions
val sessions: RDD[Session] =