chore: removing automatic vacuuming from retention policy code (#2228)

* retention policy and testcase updated

* removing dead code

* review updated code

tests/waku_archive/test_retention_policy.nim

@@ -70,11 +70,6 @@ suite "Waku Archive - Retention policy":
     let retentionPolicy: RetentionPolicy = SizeRetentionPolicy.init(size=sizeLimit)
     var putFutures = newSeq[Future[ArchiveDriverResult[void]]]()
 
-    # variables to check the db size
-    var pageSize = (waitFor driver.getPagesSize()).tryGet()
-    var pageCount = (waitFor driver.getPagesCount()).tryGet()
-    var sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
-
     # make sure that the db is empty to before test begins
     let storedMsg = (waitFor driver.getAllMessages()).tryGet()
     # if there are messages in db, empty them
@@ -95,24 +90,27 @@ suite "Waku Archive - Retention policy":
 
     ## Then
     # calculate the current database size
-    pageSize = (waitFor driver.getPagesSize()).tryGet()
-    pageCount = (waitFor driver.getPagesCount()).tryGet()
-    sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
+    let pageSize = (waitFor driver.getPagesSize()).tryGet()
+    let pageCount = (waitFor driver.getPagesCount()).tryGet()
+    let sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
+
+    # NOTE: since vacuumin is done manually, this needs to be revisited if vacuuming done automatically
+
+    # get the rows count pre-deletion
+    let rowsCountBeforeDeletion = (waitFor driver.getMessagesCount()).tryGet()
 
-    # execute policy provided the current db size oveflows 
+    # execute policy provided the current db size oveflows, results in rows deletion
     require (sizeDB >= sizeLimit)
     require (waitFor retentionPolicy.execute(driver)).isOk()
 
-    # update the current db size
-    pageSize = (waitFor driver.getPagesSize()).tryGet()
-    pageCount = (waitFor driver.getPagesCount()).tryGet()
-    sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0)
+    # get the number or rows from DB
+    let rowCountAfterDeletion = (waitFor driver.getMessagesCount()).tryGet()
 
     check:
       # size of the database is used to check if the storage limit has been preserved
       # check the current database size with the limitSize provided by the user
       # it should be lower
-      sizeDB <= sizeLimit
+      rowCountAfterDeletion <= rowsCountBeforeDeletion
 
     ## Cleanup
     (waitFor driver.close()).expect("driver to close")

waku/waku_archive/retention_policy/retention_policy_capacity.nim

@@ -62,23 +62,13 @@ method execute*(p: CapacityRetentionPolicy,
                 driver: ArchiveDriver):
                 Future[RetentionPolicyResult[void]] {.async.} =
 
-  let numMessagesRes = await driver.getMessagesCount()
-  if numMessagesRes.isErr():
-    return err("failed to get messages count: " & numMessagesRes.error)
-
-  let numMessages = numMessagesRes.value
+  let numMessages = (await driver.getMessagesCount()).valueOr:
+    return err("failed to get messages count: " & error)
 
   if numMessages < p.totalCapacity:
     return ok()
 
-  let res = await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow)
-  if res.isErr():
-    return err("deleting oldest messages failed: " & res.error)
-
-  # vacuum to get the deleted pages defragments to save storage space
-  # this will resize the database size
-  let resVaccum = await driver.performVacuum()
-  if resVaccum.isErr():
-    return err("vacuumming failed: " & resVaccum.error)
+  (await driver.deleteOldestMessagesNotWithinLimit(limit=p.capacity + p.deleteWindow)).isOkOr:
+    return err("deleting oldest messages failed: " & error)
 
   return ok()

waku/waku_archive/retention_policy/retention_policy_size.nim

@@ -50,36 +50,29 @@ method execute*(p: SizeRetentionPolicy,
   if pageSize == 0:
     return err("failed to get Page size: " & pageSizeRes.error)
 
-  # keep deleting until the current db size falls within size limit 
-  while true:
-    # to get the size of the database, pageCount and PageSize is required
-    # get page count in "messages" database
-    let pageCount = (await driver.getPagesCount()).valueOr:
-      return err("failed to get Pages count: " & $error)
-
-    # database size in megabytes (Mb)
-    let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0
-
-    if totalSizeOfDB < p.sizeLimit:
-      break
-
-    # to shread/delete messsges, get the total row/message count
-    let numMessagesRes = await driver.getMessagesCount()
-    if numMessagesRes.isErr():
-      return err("failed to get messages count: " & numMessagesRes.error)
-    let numMessages = numMessagesRes.value
-
-    # 80% of the total messages are to be kept, delete others
-    let pageDeleteWindow = int(float(numMessages) * DeleteLimit)
-
-    let res = await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow)
-    if res.isErr():
-        return err("deleting oldest messages failed: " & res.error)
-
-    # vacuum to get the deleted pages defragments to save storage space
-    # this will resize the database size
-    let resVaccum = await driver.performVacuum()
-    if resVaccum.isErr():
-      return err("vacuumming failed: " & resVaccum.error)
+  # to get the size of the database, pageCount and PageSize is required
+  # get page count in "messages" database
+  let pageCount = (await driver.getPagesCount()).valueOr:
+    return err("failed to get Pages count: " & $error)
+
+  # database size in megabytes (Mb)
+  let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0
+
+  if totalSizeOfDB < p.sizeLimit:
+    return ok()
+
+  # to shread/delete messsges, get the total row/message count
+  let numMessages = (await driver.getMessagesCount()).valueOr:
+    return err("failed to get messages count: " & error)
+
+  # NOTE: Using SQLite vacuuming is done manually, we delete a percentage of rows
+  # if vacumming is done automatically then we aim to check DB size periodially for efficient
+  # retention policy implementation.
+
+  # 80% of the total messages are to be kept, delete others
+  let pageDeleteWindow = int(float(numMessages) * DeleteLimit)
+
+  (await driver.deleteOldestMessagesNotWithinLimit(limit=pageDeleteWindow)).isOkOr:
+    return err("deleting oldest messages failed: " & error)
 
   return ok()