feat(ProjectDatabase): implement safe save functionality to prevent data loss

Notes.md

@@ -2,7 +2,7 @@
 
 Notes used during the development of the module
 
-## Save-ProjectDatabase
+## Save-ProjectDatabaseSafe
 
 ```mermaid
 graph TD;
@@ -11,7 +11,7 @@ graph TD;
 
     subgraph Module
         0([Sync-ProjectItemStaged])
-        1[Save-ProjectDatabase]
+        1[Save-ProjectDatabaseSafe]
         2[Get-Project]
         3[Get-Project]
         4[Set-ProjectV2Item]
@@ -46,7 +46,7 @@ graph TD;
     U[Update-ProjectDatabase]
     G[Invoke-GitHubOrgProjectWithFields]
 
-    U<---->V2[Set-ProjectDatabaseV2]
+    U<---->V2[Save-ProjectV2toDatabase]
 
     end
     I<-->G

Test/public/projectDatabase/project_database.test.ps1

@@ -0,0 +1,69 @@
+
+function Test_SaveProjectDatabase_SafeId_Flag_PrivateCall{
+
+    Reset-InvokeCommandMock
+    Mock_DatabaseRoot
+
+    $Owner = "SomeOrg" ; $ProjectNumber = 164
+    MockCall_GitHubOrgProjectWithFields -Owner $owner -ProjectNumber $projectNumber -FileName 'projectV2.json'
+
+    # Call on private mode
+    $modulePath = $MODULE_PATH | Split-Path -Parent
+    $module = Import-Module -Name $modulePath -PassThru
+
+    & $module {
+        $Owner = "SomeOrg" ; $ProjectNumber = 164
+        # Cache the project
+        $prj = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber
+
+
+        $prj = Get-ProjectFromDatabase -Owner $Owner -ProjectNumber $ProjectNumber
+        Assert-IsNotNull -Object $prj.safeId
+
+        # Update project and the safeId should be updated
+        $oldSafeId = $prj.safeId
+        Save-ProjectDatabaseSafe -Database $prj
+        $prj2 = Get-ProjectFromDatabase -Owner $Owner -ProjectNumber $ProjectNumber
+
+        Assert-IsNotNull -Object $prj2.safeId
+        Assert-AreNotEqual -Presented $oldSafeId -Expected $prj2.safeId
+    }
+
+}
+
+function Test_SaveProjectDatabase_Safe_PrivateCall{
+
+    Reset-InvokeCommandMock
+    Mock_DatabaseRoot
+
+    $Owner = "SomeOrg" ; $ProjectNumber = 164
+    MockCall_GitHubOrgProjectWithFields -Owner $owner -ProjectNumber $projectNumber -FileName 'projectV2.json'
+
+    # Call on private mode
+    $modulePath = $MODULE_PATH | Split-Path -Parent
+    $module = Import-Module -Name $modulePath -PassThru
+
+    & $module {
+
+        $Owner = "SomeOrg" ; $ProjectNumber = 164
+        # Cache the project
+        $prj1 = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber
+
+        # modify the project
+        $db = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber ; Save-ProjectDatabaseSafe -Database $db
+
+        # Check that safeId has changed
+        $prj2 = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber
+        Assert-AreNotEqual -Presented $prj2.safeId -Expected $prj1.safeId
+
+        ## When saving again as prj1 that has been saved before it will throw
+        $hasThrow = $false
+        try{
+            Save-ProjectDatabaseSafe -Database $prj1
+        } catch {
+            $hasThrow = $true
+        }
+        Assert-IsTrue -Condition $hasThrow
+    }
+
+}

Test/public/project_item.test.ps1

@@ -358,9 +358,6 @@ function Test_GetItemDirect_SUCCESS{
     Assert-AreEqual -Expected $contentId -Presented $result.contentId
 
 }
-function Test_GetItemDirect_SUCCESS_WithCache{
-    Assert-NotImplemented
-}
 
 function Test_ShowProjectItem_SUCCESS{
 

private/projectDatabase/project_database.ps1

@@ -68,7 +68,7 @@
     Reset-Database -Key $dbKey
 }
 
-function Set-ProjectDatabaseV2{
+function Save-ProjectV2toDatabase{
     [CmdletBinding()]
     param(
         [Parameter(Position = 0)][object]$ProjectV2,
@@ -95,30 +95,49 @@
     $db.items = $items
     $db.fields = $fields
 
+    # This is the only Save.ProjectDatabase that should not be called with -Safe
     Save-ProjectDatabase -Database $db
 }
 
+function Save-ProjectDatabaseSafe{
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory,Position = 0)][hashtable]$Database
+    )
+    Save-ProjectDatabase -Database $Database -Safe
+}
+
 function Save-ProjectDatabase{
     [CmdletBinding()]
     param(
-        [Parameter(Position = 0)][hashtable]$Database
+        [Parameter(Mandatory,Position = 0)][hashtable]$Database,
+        [Parameter()][switch]$Safe
     )
 
     $owner = $Database.owner
     $projectnumber = $Database.number
 
-    if($null -eq $Database){
-        throw "Database parameter is required"
-    }
-
     if([string]::IsNullOrWhiteSpace($owner)){
         throw "Database.owner is null or empty"
     }
     if($projectnumber -le 0){
         throw "Database.number is null or not a positive integer"
     }
-
+    
     $dbkey = Get-DatabaseKey -Owner $owner -ProjectNumber $projectnumber
+
+    if($Safe){
+        $oldDatabase = Get-Database -Key $dbkey
+
+        if ($oldDatabase.safeId -ne $Database.safeId){
+            throw "The database has changed since it was read. Aborting save to prevent overwriting changes."
+        }
+
+    }
+
+    # Add safe mark
+    $Database.safeId = [guid]::NewGuid().ToString()
+
     Save-Database -Key $dbkey -Database $Database
 }
 

private/projectDatabase/project_database_Async.ps1

@@ -22,7 +22,7 @@ function Sync-ProjectDatabaseAsync {
     $db = Sync-ProjectAsync -Database $db -SyncBatchSize $SyncBatchSize
 
     # Saved changes to database
-    Save-ProjectDatabase -Database $db
+    Save-ProjectDatabaseSafe -Database $db
 
     if (Test-ProjectDatabaseStaged -Owner $Owner -ProjectNumber $ProjectNumber) {
         "Still pending staged values" | Write-MyError
@@ -100,8 +100,6 @@ function Sync-ProjectAsync {
         Remove-ItemStaged -Database $db -ItemId $call.itemId -FieldId $call.FieldId
     }
 
-    Save-ProjectDatabase -Database $db
-
     return $db
 }
 

private/projectDatabase/project_database_Item_Resolve.ps1

@@ -0,0 +1,36 @@
+function Resolve-ProjectItem {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory, Position = 0)][object]$Database,
+        [Parameter(Mandatory, Position = 1)][string]$ItemId,
+        [Parameter()][switch]$Force
+    )
+
+    $dirty = $false
+
+    # Get Item to know what we are updating
+    $item = Get-Item -Database $Database -ItemId $ItemId
+
+    if (( ! $item ) -or $Force) {
+        "Fetching item [$ItemId] from API" | Write-Verbose
+
+        # Get direct. No cache as we are in a database modification context
+        $item = Get-ProjectItemDirect -ItemId $ItemId
+
+        if ( ! $item ) {
+            "Item [$ItemId] not found in API" | Write-MyError
+            return $null, $false
+        }
+
+        # Add to database
+        Set-Item $Database $item
+
+        # Get item again to allow the merge between staged and project fields
+        $item = Get-Item $Database $itemId
+
+        $dirty = $true
+    }
+
+    return $item, $dirty
+
+}

private/projectDatabase/project_database_Sync.ps1

@@ -20,7 +20,7 @@ function Sync-ProjectDatabase{
     # Send update to project
     $db = Sync-Project -Database $db
 
-    Save-ProjectDatabase -Database $db
+    Save-ProjectDatabaseSafe -Database $db
 
     if (Test-ProjectDatabaseStaged -Owner $Owner -ProjectNumber $ProjectNumber) {
         "Still pending staged values" | Write-MyError

private/projectDatabase/project_database_call.ps1

@@ -94,8 +94,8 @@ function GetItemInfo {
     # TODO: Situation where we are updating an item that is not cached and therefore type is not known
     if (-not $item.type) {
         # Single fetch item from api
-        # Avoid caching as we are beeing called from a Database modification context
-        $item = Get-ProjectItemDirect -ItemId $ItemId
+        # memory Database is updated wit the item too
+        $item = Resolve-ProjectItem -Database $Database -ItemId $ItemId
     }
 
     if ( $null -eq $item) {

private/projectDatabase/project_database_update.ps1

@@ -83,8 +83,8 @@ function Update-ProjectDatabase {
     # Add content fields
     $fields = $fields | Set-ContentFields
 
-    # Set-ProjectDatabase -Owner $Owner -ProjectNumber $ProjectNumber -Items $items -Fields $fields
-    Set-ProjectDatabaseV2 $projectV2 -Items $items -Fields $fields
+    # Save ProjectV2 object to ProjectDatabase
+    Save-ProjectV2toDatabase $projectV2 -Items $items -Fields $fields
 
     return $true
 }

public/project_item.ps1

@@ -27,57 +27,43 @@ function Get-ProjectItem {
         # Get Item from Project database
         $db = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber
 
+        if(! $db){ "Project not found for Owner [$Owner] and ProjectNumber [$ProjectNumber]" | Write-MyError; return $null}
+
         # Durty flag
         $durty = $false
     }
 
     process {
-        if ($db) {
-            $item = Get-Item $db $itemId
-        }
-
-        if (( ! $item ) -or $Force) {
-            "Fetching item [$ItemId] from API" | Write-Verbose
-
-            # Get direct. No cache as we are in a database modification context
-            $item = Get-ProjectItemDirect -ItemId $ItemId
-
-            # Add to database
-            Set-Item $db $item
-            $durty = $true
-
-            # Get item again to allow the merge between staged and project fields
-            $item = Get-Item $db $itemId
-        }
+        $item, $dirty = Resolve-ProjectItem -Database $db -ItemId $ItemId -Force:$Force
 
         return $item
     }
 
     end {
-        if ($durty) {
-            "Saving durty database" | Write-Verbose
-            Save-ProjectDatabase -Database $db
+        if ($dirty) {
+            "Saving dirty database" | Write-Verbose
+            Save-ProjectDatabaseSafe -Database $db
         }
     }
 
 } Export-ModuleMember -Function Get-ProjectItem
 
-function Set-ProjectItem {
-    [CmdletBinding()]
-    param(
-        [Parameter(Mandatory, ValueFromPipeline, Position = 0)][object]$Item,
-        [Parameter()][string]$Owner,
-        [Parameter()][string]$ProjectNumber
-    )
-    ($Owner, $ProjectNumber) = Get-OwnerAndProjectNumber -Owner $Owner -ProjectNumber $ProjectNumber
-    if ([string]::IsNullOrWhiteSpace($owner) -or [string]::IsNullOrWhiteSpace($ProjectNumber)) { "Owner and ProjectNumber are required" | Write-MyError; return $null }
-
-    $db = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber
-
-    Set-Item $db $item
-
-    Save-ProjectDatabase -Database $db
-}
+# function Set-ProjectItem {
+#     [CmdletBinding()]
+#     param(
+#         [Parameter(Mandatory, ValueFromPipeline, Position = 0)][object]$Item,
+#         [Parameter()][string]$Owner,
+#         [Parameter()][string]$ProjectNumber
+#     )
+#     ($Owner, $ProjectNumber) = Get-OwnerAndProjectNumber -Owner $Owner -ProjectNumber $ProjectNumber
+#     if ([string]::IsNullOrWhiteSpace($owner) -or [string]::IsNullOrWhiteSpace($ProjectNumber)) { "Owner and ProjectNumber are required" | Write-MyError; return $null }
+#
+#     $db = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber
+#
+#     Set-Item $db $item
+#
+#     Save-ProjectDatabaseSafe -Database $db
+# }
 
 function Remove-ProjectItem {
     [CmdletBinding()]
@@ -99,7 +85,7 @@ function Remove-ProjectItem {
     }
 
     end {
-        Save-ProjectDatabase -Database $db
+        Save-ProjectDatabaseSafe -Database $db
     }
 
 }
@@ -253,8 +239,8 @@ function Edit-ProjectItem {
     $db = Get-Project -Owner $Owner -ProjectNumber $ProjectNumber -Force:$Force -SkipItems:$(-not $Force)
 
     # Find the actual value of the item. Item+Staged
-    $item = Get-ProjectItem -ItemId $ItemId -Owner $Owner -ProjectNumber $ProjectNumber
-
+    # $item = Get-ProjectItem -ItemId $ItemId -Owner $Owner -ProjectNumber $ProjectNumber
+     ($item, $dirty) = Resolve-ProjectItem -Database $db -ItemId $ItemId
 
     # if the item is not found
     if($null -eq $item){ "Item [$ItemId] not found" | Write-MyError; return $null}
@@ -269,7 +255,7 @@ function Edit-ProjectItem {
     Save-ItemFieldValue $db $itemId $FieldName $Value
 
     # Commit changes to the database
-    Save-ProjectDatabase -Database $db
+    Save-ProjectDatabaseSafe -Database $db
 
 } Export-ModuleMember -Function Edit-ProjectItem
 
@@ -322,9 +308,9 @@ function Add-ProjectItemDirect {
 
                 $item = $item | Convert-NodeItemToHash
 
-                Set-ProjectItem -Owner $Owner -ProjectNumber $ProjectNumber -Item $item
+                Set-Item $db $item
 
-                Save-ProjectDatabase -Database $db
+                Save-ProjectDatabaseSafe -Database $db
 
             }
 
@@ -385,12 +371,10 @@ function Remove-ProjectItemDirect {
 
         $ret = $response.data.deleteProjectV2Item.deletedItemId
 
-        if (! $NoCache) {
-            # Remove item from cache
-            "Removing item [$ItemId] from cache" | Write-Verbose
-            Remove-Item $db $ItemId
-            Save-ProjectDatabase $db
-        }
+        # Remove item from cache
+        "Removing item [$ItemId] from cache" | Write-Verbose
+        Remove-Item $db $ItemId
+        Save-ProjectDatabaseSafe -Database $db
 
         return $ret
     }