feat: Foreach in Pagination Result

MongoCamp · Mar 31, 2023 · 1fe1d8b · 1fe1d8b
1 parent 5f3134d
commit 1fe1d8b
Show file tree

Hide file tree

Showing 14 changed files with 177 additions and 33 deletions.
diff --git a/build.sbt b/build.sbt
@@ -81,7 +81,9 @@ libraryDependencies += "org.xerial.snappy" % "snappy-java" % "1.1.9.1" % Provide
 
 libraryDependencies += "com.github.luben" % "zstd-jni" % "1.5.4-2" % Provided
 
+// #region lucene-dependency
 libraryDependencies += "org.apache.lucene" % "lucene-queryparser" % "9.5.0" % Provided
+// #endregion lucene-dependency
 
 val MongoJavaServerVersion = "1.43.0"
 

diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts
@@ -57,8 +57,7 @@ function nav() {
                 {text: 'MongoDAO', link: '/documentation/mongo-dao/'},
                 {text: 'GridFsDAO', link: '/documentation/gridfs-dao/'},
                 {text: 'Collection', link: '/documentation/collection/'},
-                {text: 'LocalServer', link: '/documentation/local-server'},
-                // { text: 'Configuration', link: '/guide/configuration' },
+                {text: 'LocalServer', link: '/documentation/local-server'}
             ]
         },
         {
@@ -86,12 +85,13 @@ function sidebarDocumentation() {
             collapsible: true,
             collapsed: true,
             items: [
-                { text: 'Introduction', link: '/documentation/database/' },
-                { text: 'Mongo Config', link: 'documentation/database/config' },
-                { text: 'DatabaseProvider', link: 'documentation/database/provider' },
-                { text: 'Reactive Streams', link: 'documentation/database/reactive-streams' },
-                { text: 'Bson', link: 'documentation/database/bson' },
-                { text: 'Relationships', link: 'documentation/database/relationships' },
+                {text: 'Introduction', link: '/documentation/database/'},
+                {text: 'Mongo Config', link: 'documentation/database/config'},
+                {text: 'DatabaseProvider', link: 'documentation/database/provider'},
+                {text: 'Reactive Streams', link: 'documentation/database/reactive-streams'},
+                {text: 'Bson', link: 'documentation/database/bson'},
+                {text: 'Relationships', link: 'documentation/database/relationships'},
+                {text: 'Lucene Query', link: '/documentation/database/lucene'}
             ]
         },
         {
@@ -100,10 +100,10 @@ function sidebarDocumentation() {
             collapsible: true,
             collapsed: true,
             items: [
-                { text: 'Introduction', link: '/documentation/mongo-dao/' },
-                { text: 'MongoDAO Base', link: '/documentation/mongo-dao/base' },
-                { text: 'CRUD Functions', link: '/documentation/mongo-dao/crud' },
-                { text: 'Search Functions', link: '/documentation/mongo-dao/search' }
+                {text: 'Introduction', link: '/documentation/mongo-dao/'},
+                {text: 'MongoDAO Base', link: '/documentation/mongo-dao/base'},
+                {text: 'CRUD Functions', link: '/documentation/mongo-dao/crud'},
+                {text: 'Search Functions', link: '/documentation/mongo-dao/search'}
             ]
         },
         {
@@ -112,9 +112,9 @@ function sidebarDocumentation() {
             collapsible: true,
             collapsed: true,
             items: [
-                { text: 'Introduction', link: '/documentation/gridfs-dao/' },
-                { text: 'CRUD Functions', link: '/documentation/gridfs-dao/crud' },
-                { text: 'Metadata', link: '/documentation/gridfs-dao/metadata' }
+                {text: 'Introduction', link: '/documentation/gridfs-dao/'},
+                {text: 'CRUD Functions', link: '/documentation/gridfs-dao/crud'},
+                {text: 'Metadata', link: '/documentation/gridfs-dao/metadata'}
             ]
         },
         {
@@ -123,8 +123,9 @@ function sidebarDocumentation() {
             collapsible: true,
             collapsed: true,
             items: [
-                { text: 'Introduction', link: '/documentation/collection/' },
-                { text: 'Aggregation', link: '/documentation/collection/aggregation' }
+                {text: 'Introduction', link: '/documentation/collection/'},
+                {text: 'Aggregation', link: '/documentation/collection/aggregation'},
+                {text: 'Pagination', link: '/documentation/collection/pagination'}
             ]
         },
         {

diff --git a/docs/documentation/collection/pagination.md b/docs/documentation/collection/pagination.md
@@ -0,0 +1,19 @@
+# Pagination
+
+In many cases you want to have the possibility to paginate over the response of your query result. MongoCamp MongoDB Driver support the pagination over Aggregation and Filter Response. As well there is an comfort methode to have an foreach over the whole Response, but with pagination to have a lower memory footprint.  
+
+## Aggregation Pagination
+
+::: warning
+The Pagination over an aggregation pipeline supports only the response of `Document`, also if you use an case class MongoDAO you will got an `Document` back. 
+:::
+
+<<< @/../src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationAggregationSpec.scala#aggregation-pagination
+
+## Find Pagination
+
+<<< @/../src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationFilterSpec.scala#filter-pagination
+
+## Foreach over Pagination result
+
+<<< @/../src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationIterationSpec.scala#aggregation-foreach
diff --git a/docs/documentation/database/lucene.md b/docs/documentation/database/lucene.md
@@ -0,0 +1,21 @@
+# Lucene Query
+
+MongoCamp Mongo Driver support the usage of [Lucene Query](https://lucene.apache.org/) to search in the MongoDb.
+
+## Usage
+Fist you have the add the lucene-queryparser Dependency to your build.sbt (without the provided marker).
+
+<<< @/../build.sbt#lucene-dependency{scala}
+
+### Explicit Usage
+The LuceneConverter has the methods to parse a String to and `Query` and a other to the document conversion.
+
+<<< @/../src/test/scala/dev/mongocamp/driver/mongodb/lucene/LuceneSearchSpec.scala#lucene-parser-with-explicit
+
+### Implicit Usage
+Like the Map to Bson conversion there is also an implicit method to convert `Query` to find Bson. 
+
+<<< @/../src/test/scala/dev/mongocamp/driver/mongodb/lucene/LuceneSearchSpec.scala#lucene-parser-with-implicit
+
+## Read More
+[Lucene Cheatsheet](https://www.lucenetutorial.com/lucene-query-syntax.html)
diff --git a/docs/index.md b/docs/index.md
@@ -21,7 +21,15 @@ features:
     details: Easy Database Config with provider and MongoConfig
   - title: DAO Pattern
     details: Implement the DAO Pattern for simple MongoDB usage [MongoDAO.
-  - title: GridFS Support
+  - title: Pagination
+    details: Use Pagination in your MongoDB for a lower RAM needing over large responses.
   - title: Enhanced BSON
+    details: Implicit Conversion from Scala Map to BSON 
+  - title: GridFS Support
+    details: It provides easy upload, download and metadata handling.
   - title: Reactive Streams
+    details: The MongoDB Scala driver is built upon Reactive Streams.
   - title: Relationships
+    details: Sometimes there is a need for relationsips beetween collections, now you can have it.
+  - title: Lucene Query
+    details: Use Apache Lucene Query Language in MongoDb
diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/Sort.scala b/src/main/scala/dev/mongocamp/driver/mongodb/Sort.scala
@@ -1,16 +1,18 @@
 package dev.mongocamp.driver.mongodb
 
 import org.bson.conversions.Bson
-import org.mongodb.scala.model.Sorts.{ ascending, descending, orderBy }
+import org.mongodb.scala.model.Sorts.{ascending, descending, orderBy}
 
 object Sort extends Sort
 
 trait Sort {
 
   def sortByKey(key: String, sortAscending: Boolean = true): Bson =
-    if (sortAscending)
+    if (sortAscending) {
       orderBy(ascending(key))
-    else
+    }
+    else {
       orderBy(descending(key))
+    }
 
 }
diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/operation/Search.scala b/src/main/scala/dev/mongocamp/driver/mongodb/operation/Search.scala
@@ -7,7 +7,7 @@ import org.bson.BsonValue
 import org.mongodb.scala.bson.ObjectId
 import org.mongodb.scala.bson.conversions.Bson
 import org.mongodb.scala.model.Filters._
-import org.mongodb.scala.{ AggregateObservable, DistinctObservable, Document, FindObservable, MongoCollection }
+import org.mongodb.scala.{AggregateObservable, DistinctObservable, Document, FindObservable, MongoCollection}
 
 import scala.reflect.ClassTag
 
@@ -21,10 +21,12 @@ abstract class Search[A]()(implicit ct: ClassTag[A]) extends Base[A] {
       projection: Bson = Document(),
       limit: Int = 0
   ): FindObservable[A] =
-    if (limit > 0)
+    if (limit > 0) {
       coll.find(filter).sort(sort).projection(projection).limit(limit)
-    else
+    }
+    else {
       coll.find(filter).sort(sort).projection(projection)
+    }
 
   def findById(oid: ObjectId): FindObservable[A] = find(equal(DatabaseProvider.ObjectIdKey, oid))
 

diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPaginatedAggregation.scala b/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPaginatedAggregation.scala
@@ -12,14 +12,14 @@ import scala.jdk.CollectionConverters._
 case class MongoPaginatedAggregation[A <: Any](
     dao: MongoDAO[A],
     aggregationPipeline: List[Bson] = List(),
-    allowDiskUse: Boolean = false,
-) {
+    allowDiskUse: Boolean = false
+) extends MongoPagination[Document] {
 
   private val AggregationKeyMetaData      = "metadata"
   private val AggregationKeyData          = "data"
   private val AggregationKeyMetaDataTotal = "total"
 
-  def paginate(page: Long, rows: Long): PaginationResult[org.bson.BsonDocument] = {
+  def paginate(page: Long, rows: Long): PaginationResult[Document] = {
     if (rows <= 0) {
       throw MongoCampPaginationException("rows per page must be greater then 0.")
     }
@@ -41,7 +41,7 @@ case class MongoPaginatedAggregation[A <: Any](
 
     val count: Long = dbResponse.get(AggregationKeyMetaData).get.asArray().get(0).asDocument().get(AggregationKeyMetaDataTotal).asNumber().longValue()
     val allPages    = Math.ceil(count.toDouble / rows).toInt
-    val list        = dbResponse.get("data").get.asArray().asScala.map(_.asDocument())
+    val list        = dbResponse.get("data").get.asArray().asScala.map(_.asDocument()).map(bdoc => Document(bdoc) )
     PaginationResult(list.toList, PaginationInfo(count, rows, page, allPages))
   }
 

diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPaginatedFilter.scala b/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPaginatedFilter.scala
@@ -4,7 +4,7 @@ import dev.mongocamp.driver.mongodb.exception.MongoCampPaginationException
 import dev.mongocamp.driver.mongodb.{MongoDAO, _}
 import org.mongodb.scala.bson.conversions.Bson
 
-case class MongoPaginatedFilter[A <: Any](dao: MongoDAO[A], filter: Bson = Map(), sort: Bson = Map(), projection: Bson = Map()) {
+case class MongoPaginatedFilter[A <: Any](dao: MongoDAO[A], filter: Bson = Map(), sort: Bson = Map(), projection: Bson = Map()) extends MongoPagination[A] {
 
   def paginate(page: Long, rows: Long): PaginationResult[A] = {
     val count = countResult

diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPagination.scala b/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPagination.scala
@@ -0,0 +1,20 @@
+package dev.mongocamp.driver.mongodb.pagination
+
+trait MongoPagination[A <: Any] {
+  def paginate(page: Long, rows: Long): PaginationResult[A]
+  def countResult: Long
+}
+object MongoPagination {
+  def foreach[A <: Any](pagination: MongoPagination[A], rows: Int = 50)(a: A => Unit): Unit = {
+    var currentPageNumber = 1
+    val rowsPerPage = if (rows < 1) Int.MaxValue else rows
+    val maxPages    = Math.ceil(pagination.countResult.toDouble / rowsPerPage).toInt
+    while (currentPageNumber > 0 && rowsPerPage > 0 && currentPageNumber <= maxPages) {
+      val page = pagination.paginate(currentPageNumber, rowsPerPage)
+      page.databaseObjects.foreach(a)
+      currentPageNumber += 1
+    }
+
+  }
+
+}
diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/lucene/LuceneSearchSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/lucene/LuceneSearchSpec.scala
@@ -10,16 +10,20 @@ class LuceneSearchSpec extends PersonSpecification {
   "LuceneSearch" should {
 
     "search with extended query" in {
+      // #region lucene-parser-with-explicit
       val luceneQuery = LuceneQueryConverter.parse("(favoriteFruit:\"apple\" AND age:\"25\") OR name:*Cecile* AND -active:false AND 123", "id")
       val search      = PersonDAO.find(LuceneQueryConverter.toDocument(luceneQuery), sortByBalance).resultList()
+      // #endregion lucene-parser-with-explicit
       search must haveSize(1)
       search.head.age mustEqual 25
       search.head.name mustEqual "Terra Salinas"
     }
 
     "search with extended query use implicit" in {
+      // #region lucene-parser-with-implicit
       val luceneQuery = LuceneQueryConverter.parse("(favoriteFruit:\"apple\" AND age:\"25\") OR name:*Cecile* AND -active:false AND 123", "id")
       val search      = PersonDAO.find(luceneQuery, sortByBalance).resultList()
+      // #endregion lucene-parser-with-implicit
       search must haveSize(1)
       search.head.age mustEqual 25
       search.head.name mustEqual "Terra Salinas"

diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationAggregationSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationAggregationSpec.scala
@@ -1,6 +1,5 @@
 package dev.mongocamp.driver.mongodb.pagination
 
-// #agg_imports
 import dev.mongocamp.driver.mongodb.Aggregate._
 import dev.mongocamp.driver.mongodb.bson.BsonConverter
 import dev.mongocamp.driver.mongodb.dao.PersonSpecification
@@ -13,23 +12,23 @@ import org.mongodb.scala.model.Filters.{and, equal}
 
 class PaginationAggregationSpec extends PersonSpecification {
 
-  // #agg_stages
   val filterStage: Bson = filter(and(equal("gender", "female"), notNullFilter("balance")))
 
   val groupStage: Bson = group(Map("age" -> "$age"), sumField("balance"), firstField("age"))
 
   val sortStage: Bson = sort(sortByKey("age"))
-  // #agg_stages
 
   "Search" should {
 
     "support aggregation filter" in {
 
+      // #region aggregation-pagination
       val pipeline = List(filterStage, sortStage)
 
       val pagination = MongoPaginatedAggregation(PersonDAO.Raw, pipeline, allowDiskUse = true)
 
       val page = pagination.paginate(1, 10)
+      // #endregion aggregation-pagination
 
       (pagination.countResult must be).equalTo(98)
 
@@ -44,7 +43,7 @@ class PaginationAggregationSpec extends PersonSpecification {
       // #agg_execute
       val pipeline = List(filterStage, groupStage, sortStage)
 
-      val pagination = MongoPaginatedAggregation(PersonDAO.Raw, pipeline, allowDiskUse = true)
+      val pagination = MongoPaginatedAggregation(PersonDAO, pipeline, allowDiskUse = true)
 
       val page = pagination.paginate(1, 10)
 

diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationFilterSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationFilterSpec.scala
@@ -28,9 +28,11 @@ class PaginationFilterSpec extends PersonSpecification with MongoImplicits {
     }
 
     "support with Filter" in {
+      // #region filter-pagination
       val paginationFemale = MongoPaginatedFilter(PersonDAO, Map("gender" -> "female"), sortByKey("name"))
 
       val pageFemale = paginationFemale.paginate(1, 10)
+      // #endregion filter-pagination
 
       paginationFemale.countResult mustEqual 11
       pageFemale.paginationInfo.pagesCount mustEqual 10

diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationIterationSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/pagination/PaginationIterationSpec.scala
@@ -0,0 +1,64 @@
+package dev.mongocamp.driver.mongodb.pagination
+
+import com.typesafe.scalalogging.LazyLogging
+import dev.mongocamp.driver.MongoImplicits
+import dev.mongocamp.driver.mongodb.Aggregate._
+import dev.mongocamp.driver.mongodb._
+import dev.mongocamp.driver.mongodb.dao.PersonSpecification
+import dev.mongocamp.driver.mongodb.test.TestDatabase._
+import org.mongodb.scala.bson.conversions.Bson
+import org.mongodb.scala.model.Aggregates.{filter, sort}
+import org.mongodb.scala.model.Filters.{and, equal}
+class PaginationIterationSpec extends PersonSpecification with MongoImplicits with LazyLogging{
+
+  "Pagination Iteration" should {
+
+    "support with Filter" in {
+      // #region filter-foreach
+      val paginationFemale = MongoPaginatedFilter(PersonDAO, Map("gender" -> "female"), sortByKey("name"))
+
+      val pageFemale = paginationFemale.paginate(1, 10)
+
+      pageFemale.paginationInfo.allCount mustEqual 98
+
+      var i = 0
+
+      MongoPagination.foreach(paginationFemale, 5) { person =>
+        {
+          logger.trace(person.toString)
+          i = i + 1
+        }
+      }
+      i mustEqual 98
+      // #endregion filter-foreach
+
+    }
+
+    "support with aggregation" in {
+      // #region aggregation-foreach
+      val filterStage: Bson = filter(and(equal("gender", "female"), notNullFilter("balance")))
+
+      val sortStage: Bson = sort(sortByKey("age"))
+
+      val pipeline = List(filterStage, sortStage)
+
+      val pagination = MongoPaginatedAggregation(PersonDAO, pipeline, allowDiskUse = true)
+
+      val page = pagination.paginate(1, 10)
+
+      page.paginationInfo.allCount mustEqual 98
+      var i = 0
+      MongoPagination.foreach(pagination, 5) { element =>
+        {
+          logger.trace(element.toJson())
+          i = i + 1
+        }
+      }
+      i mustEqual 98
+      // #endregion aggregation-foreach
+
+    }
+
+  }
+
+}