From 5a37fc947d090af244d52b6b5c8f6eb1bf9446ca Mon Sep 17 00:00:00 2001 From: zetashift Date: Fri, 10 Mar 2023 18:26:41 +0100 Subject: [PATCH 01/10] Initial example for fs2-data-csv for Scala 3 --- docs/examples.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/docs/examples.md b/docs/examples.md index 87292f6..f683ceb 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -164,9 +164,79 @@ object Main extends CommandIOApp("mkString", "Concatenates strings from stdin") @:@ +## Parsing and transforming a CSV file + +Here, [fs2-data-csv] is used to read and parse a comma separated file. +Manual encoders and decoders are defined for our `Passenger`s to showcase `fs2-data` power. + + +@:select(scala-version) + +@:choice(scala-3) +```scala mdoc:reset:silent +//> using lib "org.typelevel::toolkit::@VERSION@" + +import cats.effect.* +import fs2.text +import fs2.data.csv.* +import fs2.data.csv.generic.semiauto._ +import fs2.io.file.{Path, Flags, Files} +import cats.data.NonEmptyList + +case class Passenger( + id: Long, + firstName: String, + age: Either[String, Int], + flightNumber: String, + country: String +) + +object Passenger: + given csvRowDecoder: CsvRowDecoder[Passenger, String] with + def apply(row: CsvRow[String]): DecoderResult[Passenger] = + for + id <- row.as[Long]("id") + firstName <- row.as[String]("First Name") + ageOpt <- row.asNonEmpty[Int]("Age") + flightNumber <- row.as[String]("flight number") + country <- row.as[String]("country") + yield + val age = ageOpt.toRight[String]("N/A") + Passenger(id, firstName, age, flightNumber, country) + + given csvRowEncoder: CsvRowEncoder[Passenger, String] with + def apply(p: Passenger): CsvRow[String] = + CsvRow.fromNelHeaders( + NonEmptyList.of( + (p.firstName, "first_name"), + (p.age.toString(), "age"), + (p.flightNumber, "flight_number"), + (p.country, "country") + ) + ) + +val input = Files[IO] + .readAll(Path("./example.csv"), 1024, Flags.Read) + .through(text.utf8.decode) + .through(decodeUsingHeaders[Passenger]()) + +object CSVPrinter extends IOApp.Simple: + val run = + input.evalTap(p => IO.println(p)).compile.drain +``` + + +@:choice(scala-2) +```scala mdoc:reset:silent +// TODO +``` +@:@ + [fs2]: https://fs2.io/#/ +[fs2-data-csv]: https://fs2-data.gnieh.org/documentation/csv/ [decline]: https://ben.kirw.in/decline/ [scala-native]: https://scala-native.org/en/stable/ [Scala CLI]: https://scala-cli.virtuslab.org/ [Koroeskohr]: https://github.com/Koroeskohr + From 337fd362705a2d092769f368e27930551fc5c71f Mon Sep 17 00:00:00 2001 From: zetashift Date: Fri, 10 Mar 2023 18:26:41 +0100 Subject: [PATCH 02/10] Improve doc for the example --- docs/examples.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/examples.md b/docs/examples.md index f683ceb..3946b19 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -167,7 +167,16 @@ object Main extends CommandIOApp("mkString", "Concatenates strings from stdin") ## Parsing and transforming a CSV file Here, [fs2-data-csv] is used to read and parse a comma separated file. -Manual encoders and decoders are defined for our `Passenger`s to showcase `fs2-data` power. +Manual encoders and decoders are defined for our `Passenger`s to show you how to everything from scratch. + +Let's start with a CSV file that has records of fictious passengers registered for a flight: + +``` +id,First Name,Age,flight number,country +1,Seyton,44,WX122,Tanzania +2,Lina,,UX199,Greenland +3,Grogu,,SW999,Singapore +``` @:select(scala-version) From 35c1234ccacb7f82419cfff2989028f68a46b561 Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 17:17:30 +0100 Subject: [PATCH 03/10] Improve fs2-data example and add Scala 2 version --- docs/examples.md | 77 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 3946b19..675d79e 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -167,12 +167,12 @@ object Main extends CommandIOApp("mkString", "Concatenates strings from stdin") ## Parsing and transforming a CSV file Here, [fs2-data-csv] is used to read and parse a comma separated file. -Manual encoders and decoders are defined for our `Passenger`s to show you how to everything from scratch. +Manual encoders and decoders are defined for our `Passenger`s to show you how to do everything from scratch. Let's start with a CSV file that has records of fictious passengers registered for a flight: ``` -id,First Name,Age,flight number,country +id,First Name,Age,flight number,destination 1,Seyton,44,WX122,Tanzania 2,Lina,,UX199,Greenland 3,Grogu,,SW999,Singapore @@ -197,10 +197,11 @@ case class Passenger( firstName: String, age: Either[String, Int], flightNumber: String, - country: String + destination: String ) object Passenger: + // Here we define a manual decoder for each row in our CSV given csvRowDecoder: CsvRowDecoder[Passenger, String] with def apply(row: CsvRow[String]): DecoderResult[Passenger] = for @@ -208,11 +209,12 @@ object Passenger: firstName <- row.as[String]("First Name") ageOpt <- row.asNonEmpty[Int]("Age") flightNumber <- row.as[String]("flight number") - country <- row.as[String]("country") + destination <- row.as[String]("destination") yield val age = ageOpt.toRight[String]("N/A") - Passenger(id, firstName, age, flightNumber, country) + Passenger(id, firstName, age, flightNumber, destination) + // Here we define a manual encoder for encoding Passenger classes to a CSV given csvRowEncoder: CsvRowEncoder[Passenger, String] with def apply(p: Passenger): CsvRow[String] = CsvRow.fromNelHeaders( @@ -220,7 +222,7 @@ object Passenger: (p.firstName, "first_name"), (p.age.toString(), "age"), (p.flightNumber, "flight_number"), - (p.country, "country") + (p.destination, "destination") ) ) @@ -230,14 +232,73 @@ val input = Files[IO] .through(decodeUsingHeaders[Passenger]()) object CSVPrinter extends IOApp.Simple: - val run = + val run: IO[Unit] = input.evalTap(p => IO.println(p)).compile.drain ``` @:choice(scala-2) ```scala mdoc:reset:silent -// TODO + +//> using lib "org.typelevel::toolkit::0.0.2" +//> using lib "org.gnieh::fs2-data-csv::1.6.1" +//> using lib "org.gnieh::fs2-data-csv-generic::1.6.1" + +import cats.effect.* +import fs2.text +import fs2.data.csv.* +import fs2.data.csv.generic.semiauto._ +import fs2.io.file.{Path, Flags, Files} +import cats.data.NonEmptyList + +case class Passenger( + id: Long, + firstName: String, + age: Either[String, Int], + flightNumber: String, + destination: String +) + +object Passenger { + // Here we define a manual decoder for each row in our CSV + implicit val csvRowDecoder: CsvRowDecoder[Passenger, String] = + new CsvRowDecoder[Passenger, String] { + def apply(row: CsvRow[String]): DecoderResult[Passenger] = + for + id <- row.as[Long]("id") + firstName <- row.as[String]("First Name") + ageOpt <- row.asNonEmpty[Int]("Age") + flightNumber <- row.as[String]("flight number") + destination <- row.as[String]("destination") + yield + val age = ageOpt.toRight[String]("N/A") + Passenger(id, firstName, age, flightNumber, destination) + } + + // Here we define a manual encoder for encoding Passenger classes to a CSV + implicit val csvRowEncoder: CsvRowEncoder[Passenger, String] = + new CsvRowEncoder[Passenger, String] { + def apply(p: Passenger): CsvRow[String] = + CsvRow.fromNelHeaders( + NonEmptyList.of( + (p.firstName, "first_name"), + (p.age.toString(), "age"), + (p.flightNumber, "flight_number"), + (p.destination, "destination") + ) + ) + } +} + +val input = Files[IO] + .readAll(Path("./example.csv"), 1024, Flags.Read) + .through(text.utf8.decode) + .through(decodeUsingHeaders[Passenger]()) + +object CSVPrinter extends IOApp.Simple { + val run: IO[Unit] = + input.evalTap(p => IO.println(p)).compile.drain +} ``` @:@ From 42bf21330d238754031119e6f2781e7805805961 Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 17:23:59 +0100 Subject: [PATCH 04/10] Fix import for Scala 2 fs2-data example --- docs/examples.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 675d79e..d3db249 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -239,10 +239,7 @@ object CSVPrinter extends IOApp.Simple: @:choice(scala-2) ```scala mdoc:reset:silent - -//> using lib "org.typelevel::toolkit::0.0.2" -//> using lib "org.gnieh::fs2-data-csv::1.6.1" -//> using lib "org.gnieh::fs2-data-csv-generic::1.6.1" +//> using lib "org.typelevel::toolkit::@VERSION@" import cats.effect.* import fs2.text From 6618846de8f546afc9d6f8e0a8d4af18b318c809 Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 17:26:47 +0100 Subject: [PATCH 05/10] Remove double fs2-data mention in readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 5c83260..6c21a37 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,6 @@ It currently includes: * [Circe] and http4s integration * [Decline Effect] * [Munit Cats Effect] -* [fs2-data-csv] [Scala Toolkit]: https://virtuslab.com/blog/scala-toolkit-makes-scala-powerful-straight-out-of-the-box/ [Cats]: https://typelevel.org/cats @@ -32,4 +31,3 @@ It currently includes: [Circe]: https://circe.github.io/circe/ [Decline Effect]: https://ben.kirw.in/decline/effect.html [Munit Cats Effect]: https://github.com/typelevel/munit-cats-effect -[fs2-data-csv]: https://fs2-data.gnieh.org/documentation/csv/ From 92a7ddb587110ecd8ffaeaa8e09560586d7d4866 Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 18:22:42 +0100 Subject: [PATCH 06/10] Actually make the Scala 2 example decent --- docs/examples.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index d3db249..6206396 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -188,7 +188,7 @@ id,First Name,Age,flight number,destination import cats.effect.* import fs2.text import fs2.data.csv.* -import fs2.data.csv.generic.semiauto._ +import fs2.data.csv.generic.semiauto.* import fs2.io.file.{Path, Flags, Files} import cats.data.NonEmptyList @@ -241,9 +241,9 @@ object CSVPrinter extends IOApp.Simple: ```scala mdoc:reset:silent //> using lib "org.typelevel::toolkit::@VERSION@" -import cats.effect.* +import cats.effect._ import fs2.text -import fs2.data.csv.* +import fs2.data.csv._ import fs2.data.csv.generic.semiauto._ import fs2.io.file.{Path, Flags, Files} import cats.data.NonEmptyList @@ -253,7 +253,7 @@ case class Passenger( firstName: String, age: Either[String, Int], flightNumber: String, - destination: String + country: String ) object Passenger { @@ -261,15 +261,16 @@ object Passenger { implicit val csvRowDecoder: CsvRowDecoder[Passenger, String] = new CsvRowDecoder[Passenger, String] { def apply(row: CsvRow[String]): DecoderResult[Passenger] = - for + for { id <- row.as[Long]("id") firstName <- row.as[String]("First Name") ageOpt <- row.asNonEmpty[Int]("Age") flightNumber <- row.as[String]("flight number") - destination <- row.as[String]("destination") - yield + country <- row.as[String]("country") + } yield { val age = ageOpt.toRight[String]("N/A") - Passenger(id, firstName, age, flightNumber, destination) + Passenger(id, firstName, age, flightNumber, country) + } } // Here we define a manual encoder for encoding Passenger classes to a CSV @@ -281,20 +282,21 @@ object Passenger { (p.firstName, "first_name"), (p.age.toString(), "age"), (p.flightNumber, "flight_number"), - (p.destination, "destination") + (p.country, "country") ) ) } } -val input = Files[IO] - .readAll(Path("./example.csv"), 1024, Flags.Read) - .through(text.utf8.decode) - .through(decodeUsingHeaders[Passenger]()) - object CSVPrinter extends IOApp.Simple { + val input = Files[IO] + .readAll(Path("./example.csv"), 1024, Flags.Read) + .through(text.utf8.decode) + .through(decodeUsingHeaders[Passenger]()) + val run: IO[Unit] = input.evalTap(p => IO.println(p)).compile.drain + } ``` @:@ From af2fd1606455c9d4d46dfd001994b8d32f49d43c Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 18:52:59 +0100 Subject: [PATCH 07/10] Add a bit more fancy console logging to fs2-data example --- docs/examples.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 6206396..0e19263 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -232,8 +232,16 @@ val input = Files[IO] .through(decodeUsingHeaders[Passenger]()) object CSVPrinter extends IOApp.Simple: + val run: IO[Unit] = - input.evalTap(p => IO.println(p)).compile.drain + input + .evalTap(p => + IO.println( + s"${p.firstName} is taking flight: ${p.flightNumber} to ${p.destination}" + ) + ) + .compile + .drain ``` @@ -295,8 +303,14 @@ object CSVPrinter extends IOApp.Simple { .through(decodeUsingHeaders[Passenger]()) val run: IO[Unit] = - input.evalTap(p => IO.println(p)).compile.drain - + input + .evalTap(p => + IO.println( + s"${p.firstName} is taking flight: ${p.flightNumber} to ${p.destination}" + ) + ) + .compile + .drain } ``` @:@ From fe7e459df29dd5b52959da5bebb4907f28855136 Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 19:09:44 +0100 Subject: [PATCH 08/10] Fix destination field for fs2-data example --- docs/examples.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 0e19263..0b5d386 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -261,7 +261,7 @@ case class Passenger( firstName: String, age: Either[String, Int], flightNumber: String, - country: String + destination: String ) object Passenger { @@ -274,10 +274,10 @@ object Passenger { firstName <- row.as[String]("First Name") ageOpt <- row.asNonEmpty[Int]("Age") flightNumber <- row.as[String]("flight number") - country <- row.as[String]("country") + destination <- row.as[String]("destination") } yield { val age = ageOpt.toRight[String]("N/A") - Passenger(id, firstName, age, flightNumber, country) + Passenger(id, firstName, age, flightNumber, destination) } } @@ -290,7 +290,7 @@ object Passenger { (p.firstName, "first_name"), (p.age.toString(), "age"), (p.flightNumber, "flight_number"), - (p.country, "country") + (p.destination, "destination") ) ) } From 4220fb14e95bfb1a3329cd80ac15ceaf4292425b Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 20:01:02 +0100 Subject: [PATCH 09/10] Add calculating of mean ages in fs2-data example --- docs/examples.md | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 0b5d386..5001e7a 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -234,6 +234,14 @@ val input = Files[IO] object CSVPrinter extends IOApp.Simple: val run: IO[Unit] = + // Let's do an aggregation of all the values in the age column + val meanIO = + input + .foldMap(p => (p.age.getOrElse(0), 1)) + .compile + .lastOrError + .map((sum, count) => sum / count) + input .evalTap(p => IO.println( @@ -241,7 +249,9 @@ object CSVPrinter extends IOApp.Simple: ) ) .compile - .drain + .drain >> meanIO.flatMap(mean => + IO.println(s"The mean age of the passengers is $mean") + ) ``` @@ -302,7 +312,16 @@ object CSVPrinter extends IOApp.Simple { .through(text.utf8.decode) .through(decodeUsingHeaders[Passenger]()) - val run: IO[Unit] = + + val run: IO[Unit] = { + // Let's do an aggregation of all the values in the age column + val meanIO = + input + .foldMap(p => (p.age.getOrElse(0), 1)) + .compile + .lastOrError + .map((sum, count) => sum / count) + input .evalTap(p => IO.println( @@ -310,7 +329,10 @@ object CSVPrinter extends IOApp.Simple { ) ) .compile - .drain + .drain >> meanIO.flatMap(mean => + IO.println(s"The mean age of the passengers is $mean") + ) + } } ``` @:@ From 095ec121b8f4e170c0f13338a3f879406245ce88 Mon Sep 17 00:00:00 2001 From: zetashift Date: Sat, 11 Mar 2023 20:04:44 +0100 Subject: [PATCH 10/10] Fix scala 2 example for fs2-data --- docs/examples.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples.md b/docs/examples.md index 5001e7a..7c52023 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -320,7 +320,7 @@ object CSVPrinter extends IOApp.Simple { .foldMap(p => (p.age.getOrElse(0), 1)) .compile .lastOrError - .map((sum, count) => sum / count) + .map({ case (sum, count) => sum / count}) input .evalTap(p =>