feat: Compact Methods for Database or DAO

QuadStingray · QuadStingray · commit 0926b1da0ba3 · 2023-03-27T21:47:47.000+02:00
diff --git a/README.md b/README.md
@@ -154,8 +154,14 @@ class RestaurantDemoSpec extends Specification with RestaurantDemoDatabaseFuncti
   }
 
 }
+```
 
+## Run Tests
+```shell
+docker run --publish 27017:27017  mongocamp/mongodb:latest;
+sbt test;
 ```
+
 ## Supporters
 
 JetBrains is supporting this open source project with:
diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/MongoDAO.scala b/src/main/scala/dev/mongocamp/driver/mongodb/MongoDAO.scala
@@ -1,21 +1,19 @@
 package dev.mongocamp.driver.mongodb
 
-import java.nio.charset.Charset
-
 import better.files.File
-import dev.mongocamp.driver.mongodb.bson.{ BsonConverter, DocumentHelper }
-import dev.mongocamp.driver.mongodb.database.{ ChangeObserver, CollectionStatus, DatabaseProvider }
+import dev.mongocamp.driver.mongodb.bson.{BsonConverter, DocumentHelper}
+import dev.mongocamp.driver.mongodb.database.{ChangeObserver, CollectionStatus, CompactResult, DatabaseProvider}
 import dev.mongocamp.driver.mongodb.operation.Crud
 import org.bson.json.JsonParseException
-import org.mongodb.scala.model.Aggregates.project
+import org.mongodb.scala.model.Accumulators._
+import org.mongodb.scala.model.Aggregates._
+import org.mongodb.scala.model.Filters._
 import org.mongodb.scala.model.Projections
-import org.mongodb.scala.{ BulkWriteResult, Document, MongoCollection, Observable, SingleObservable }
+import org.mongodb.scala.{BulkWriteResult, Document, MongoCollection, Observable, SingleObservable}
 
+import java.nio.charset.Charset
 import scala.collection.mutable.ArrayBuffer
 import scala.reflect.ClassTag
-import org.mongodb.scala.model.Filters._
-import org.mongodb.scala.model.Aggregates._
-import org.mongodb.scala.model.Accumulators._
 
 /** Created by tom on 20.01.17.
   */
@@ -32,8 +30,13 @@ abstract class MongoDAO[A](provider: DatabaseProvider, collectionName: String)(i
     observer
   }
 
-  def collectionStatus: Observable[CollectionStatus] =
+  def collectionStatus: Observable[CollectionStatus] = {
     provider.runCommand(Map("collStats" -> collectionName)).map(document => CollectionStatus(document))
+  }
+
+  def compact: Observable[Option[CompactResult]] = {
+    provider.runCommand(Map("compact" -> collectionName)).map(document => CompactResult(document))
+  }
 
   /** @param sampleSize
     *   use sample size greater 0 for better performance on big collections
@@ -45,10 +48,12 @@ abstract class MongoDAO[A](provider: DatabaseProvider, collectionName: String)(i
     val unwindStage  = unwind("$tempArray")
     val groupStage   = group("_id", addToSet("keySet", "$tempArray.k"))
     val pipeline = {
-      if (sampleSize > 0)
+      if (sampleSize > 0) {
         List(sample(sampleSize), projectStage, unwindStage, groupStage)
-      else
+      }
+      else {
         List(projectStage, unwindStage, groupStage)
+      }
     }
 
     val aggregationResult: Document = Raw.findAggregated(pipeline).result()
@@ -62,9 +67,11 @@ abstract class MongoDAO[A](provider: DatabaseProvider, collectionName: String)(i
 
   def importJsonFile(file: File): SingleObservable[BulkWriteResult] = {
     val docs = new ArrayBuffer[Document]()
-    try if (file.exists) {
-      val iterator = file.lineIterator(Charset.forName("UTF-8"))
-      iterator.foreach(line => docs.+=(DocumentHelper.documentFromJsonString(line).get))
+    try {
+      if (file.exists) {
+        val iterator = file.lineIterator(Charset.forName("UTF-8"))
+        iterator.foreach(line => docs.+=(DocumentHelper.documentFromJsonString(line).get))
+      }
     }
     catch {
       case e: JsonParseException =>
diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/database/CompactResult.scala b/src/main/scala/dev/mongocamp/driver/mongodb/database/CompactResult.scala
@@ -0,0 +1,20 @@
+package dev.mongocamp.driver.mongodb.database
+import dev.mongocamp.driver.mongodb._
+import org.mongodb.scala.bson.Document
+
+case class CompactResult(bytesFreed: Long)
+
+object CompactResult {
+  def apply(document: Document): Option[CompactResult] = {
+    if (document.getLongValue("ok") == 1) {
+      Some(
+        CompactResult(
+          document.getLongValue("bytesFreed")
+        )
+      )
+    }
+    else {
+      None
+    }
+  }
+}
diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/database/DatabaseProvider.scala b/src/main/scala/dev/mongocamp/driver/mongodb/database/DatabaseProvider.scala
@@ -2,7 +2,7 @@ package dev.mongocamp.driver.mongodb.database
 
 import dev.mongocamp.driver.mongodb._
 import dev.mongocamp.driver.mongodb.bson.codecs.CustomCodecProvider
-import org.bson.codecs.configuration.CodecRegistries.{ fromProviders, fromRegistries }
+import org.bson.codecs.configuration.CodecRegistries.{fromProviders, fromRegistries}
 import org.bson.codecs.configuration.CodecRegistry
 import org.mongodb.scala.MongoClient.DEFAULT_CODEC_REGISTRY
 import org.mongodb.scala._
@@ -42,6 +42,16 @@ class DatabaseProvider(val config: MongoConfig, val registry: CodecRegistry) ext
 
   def dropDatabase(databaseName: String = DefaultDatabaseName): SingleObservable[Void] = database(databaseName).drop()
 
+  def compact(databaseName: String = DefaultDatabaseName, maxWaitPerCollection: Int = DefaultMaxWait): CompactResult = {
+    CompactResult(
+      collectionNames(databaseName)
+        .map(
+        collectionName => dao(collectionName).compact.result(maxWaitPerCollection).getOrElse(CompactResult(0)))
+        .map(_.bytesFreed)
+        .sum
+    )
+  }
+
   def database(databaseName: String = DefaultDatabaseName): MongoDatabase = {
     if (!cachedDatabaseMap.contains(databaseName)) {
       cachedDatabaseMap.put(databaseName, client.getDatabase(databaseName).withCodecRegistry(registry))
diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/operation/Base.scala b/src/main/scala/dev/mongocamp/driver/mongodb/operation/Base.scala
@@ -1,11 +1,11 @@
 package dev.mongocamp.driver.mongodb.operation
 
-import dev.mongocamp.driver.mongodb.database.{ ConfigHelper, MongoIndex }
 import com.typesafe.scalalogging.LazyLogging
+import dev.mongocamp.driver.mongodb.database.MongoIndex
 import org.mongodb.scala.bson.conversions.Bson
 import org.mongodb.scala.model.Sorts._
-import org.mongodb.scala.model.{ CountOptions, DropIndexOptions, IndexOptions, Indexes }
-import org.mongodb.scala.{ Document, ListIndexesObservable, MongoCollection, Observable, SingleObservable }
+import org.mongodb.scala.model.{CountOptions, DropIndexOptions, IndexOptions, Indexes}
+import org.mongodb.scala.{Document, ListIndexesObservable, MongoCollection, Observable, SingleObservable}
 
 import scala.concurrent.duration.Duration
 import scala.reflect.ClassTag
@@ -23,11 +23,14 @@ abstract class Base[A]()(implicit ct: ClassTag[A]) extends LazyLogging {
       fieldName: String,
       sortAscending: Boolean = true,
       options: IndexOptions = IndexOptions()
-  ): SingleObservable[String] =
-    if (sortAscending)
+  ): SingleObservable[String] = {
+    if (sortAscending) {
       createIndex(ascending(fieldName), options)
-    else
+    }
+    else {
       createIndex(descending(fieldName), options)
+    }
+  }
 
   def createIndexForFieldWithName(
       fieldName: String,
diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/CompactSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/CompactSpec.scala
@@ -0,0 +1,36 @@
+package dev.mongocamp.driver.mongodb
+
+import better.files.{File, Resource}
+import dev.mongocamp.driver.mongodb.database.CompactResult
+import dev.mongocamp.driver.mongodb.test.TestDatabase
+import dev.mongocamp.driver.mongodb.test.TestDatabase.BookDAO
+import org.specs2.mutable.Specification
+import org.specs2.specification.BeforeAll
+
+import java.text.SimpleDateFormat
+import java.util.Date
+
+class CompactSpec extends Specification with BeforeAll {
+  val DateFormat = new SimpleDateFormat("yyyy-MM-dd")
+  val From: Date = DateFormat.parse("2000-01-01")
+
+  override def beforeAll(): Unit = {
+    BookDAO.drop().result()
+    BookDAO.importJsonFile(File(Resource.getUrl("json/books.json"))).result()
+    val stats = BookDAO.collectionStatus.result()
+    stats.count mustEqual 431
+  }
+
+  "CompactSpec" should {
+    "compact single collection" in {
+      val count: Option[CompactResult] = BookDAO.compact.result()
+      count must beSome()
+      count.get.bytesFreed must beGreaterThanOrEqualTo(0L)
+    }
+    "compact complete database" in {
+      val count: CompactResult = TestDatabase.provider.compact()
+      count.bytesFreed must beGreaterThanOrEqualTo(0L)
+    }
+  }
+
+}
diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/dao/BookDAOSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/dao/BookDAOSpec.scala
@@ -1,22 +1,20 @@
 package dev.mongocamp.driver.mongodb.dao
 
-import java.text.SimpleDateFormat
-import java.util.Date
-
 import better.files.{File, Resource}
 import dev.mongocamp.driver.mongodb.Filter._
 import dev.mongocamp.driver.mongodb._
 import dev.mongocamp.driver.mongodb.database.DatabaseProvider
 import dev.mongocamp.driver.mongodb.test.TestDatabase.BookDAO
 import org.mongodb.scala.bson.conversions.Bson
-import org.mongodb.scala.model.Aggregates.group
-import org.mongodb.scala.model.Aggregates.filter
-import org.mongodb.scala.model.Aggregates.project
+import org.mongodb.scala.model.Aggregates.{filter, group, project}
 import org.mongodb.scala.model.Filters.and
 import org.mongodb.scala.model.Projections
+import org.mongodb.scala.model.Updates._
 import org.specs2.mutable.Specification
 import org.specs2.specification.BeforeAll
-import org.mongodb.scala.model.Updates._
+
+import java.text.SimpleDateFormat
+import java.util.Date
 
 class BookDAOSpec extends Specification with BeforeAll {
   val DateFormat = new SimpleDateFormat("yyyy-MM-dd")
diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/dao/PersonDAOSpec.scala b/src/test/scala/dev/mongocamp/driver/mongodb/dao/PersonDAOSpec.scala
@@ -1,11 +1,10 @@
 package dev.mongocamp.driver.mongodb.dao
 
-import java.util.concurrent.TimeUnit
-
 import dev.mongocamp.driver.MongoImplicits
-import dev.mongocamp.driver.mongodb.test.TestDatabase.PersonDAO
 import dev.mongocamp.driver.mongodb.model.Person
+import dev.mongocamp.driver.mongodb.test.TestDatabase.PersonDAO
 
+import java.util.concurrent.TimeUnit
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, Future}
diff --git a/src/test/scala/dev/mongocamp/driver/mongodb/test/TestDatabase.scala b/src/test/scala/dev/mongocamp/driver/mongodb/test/TestDatabase.scala
@@ -2,10 +2,10 @@ package dev.mongocamp.driver.mongodb.test
 
 import better.files.File
 import com.mongodb.client.model.changestream.OperationType
+import com.typesafe.scalalogging.LazyLogging
 import dev.mongocamp.driver.mongodb.database.DatabaseProvider
 import dev.mongocamp.driver.mongodb.model._
 import dev.mongocamp.driver.mongodb.{GridFSDAO, MongoDAO}
-import com.typesafe.scalalogging.LazyLogging
 import org.bson.codecs.configuration.CodecRegistries.{fromProviders, fromRegistries}
 import org.mongodb.scala.Document
 import org.mongodb.scala.bson.codecs.Macros._
@@ -19,20 +19,19 @@ object TestDatabase extends LazyLogging {
 
   private val registry = fromProviders(classOf[Person], classOf[Friend], classOf[CodecTest], classOf[Book])
 
-  val provider =
-    DatabaseProvider.fromPath(configPath = "unit.test.mongo", registry = fromRegistries(registry))
-
-  // provider.addChangeObserver(ChangeObserver(consumeDatabaseChanges))
+  val provider = DatabaseProvider.fromPath(configPath = "unit.test.mongo", registry = fromRegistries(registry))
 
-  def consumeDatabaseChanges(changeStreamDocument: ChangeStreamDocument[Document]): Unit =
-    if (changeStreamDocument.getOperationType != OperationType.INSERT)
+  def consumeDatabaseChanges(changeStreamDocument: ChangeStreamDocument[Document]): Unit = {
+    if (changeStreamDocument.getOperationType != OperationType.INSERT) {
       logger.info(
         "changed %s:%s with ID: %s".format(
           changeStreamDocument.getNamespace,
           changeStreamDocument.getOperationType,
           changeStreamDocument.getDocumentKey
         )
       )
+    }
+  }
 
   object PersonDAO extends MongoDAO[Person](provider, "people")
 
diff --git a/version.sbt b/version.sbt
@@ -1,11 +1,5 @@
-import scala.io.Source
-import scala.tools.nsc.io.File
+import dev.quadstingray.sbt.json.JsonFile
 
-ThisBuild / version := {
-  val packageJsonFile   = File("package.json")
-  val source            = Source.fromFile(packageJsonFile.toURI)
-  val versionPattern    = "\"version\":(.*?)\"(.*?)\"".r
-  val versionPartString = versionPattern.findFirstIn(source.mkString).get
-  val replacedVersion   = versionPartString.replace("\"version\":", "").replace("\"", "").replace("\",", "").trim.trim.trim
-  replacedVersion.toLowerCase.replace(".snapshot", "-SNAPSHOT")
-}
+val json = JsonFile(file("package.json"))
+
+ThisBuild / version := json.stringValue("version")

Original file line number	Diff line number	Diff line change
`@@ -154,8 +154,14 @@ class RestaurantDemoSpec extends Specification with RestaurantDemoDatabaseFuncti`
`154`	`154`	`}`
`155`	`155`
`156`	`156`	`}`
	`157`	+```
`157`	`158`
	`159`	`+## Run Tests`
	`160`	+```shell
	`161`	`+docker run --publish 27017:27017 mongocamp/mongodb:latest;`
	`162`	`+sbt test;`
`158`	`163`	```
	`164`	`+`
`159`	`165`	`## Supporters`
`160`	`166`
`161`	`167`	`JetBrains is supporting this open source project with:`