From 53ddd758a7fbf41861481133623eb74fb9819655 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Sun, 10 Jan 2021 07:16:14 +1100 Subject: [PATCH 1/3] Catch database assertion when item path cannot be calculated * Catch database assertion when item path cannot be calculated --- src/itemdb.d | 3 +++ src/sync.d | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/itemdb.d b/src/itemdb.d index 1c929fe84..91972d2ce 100644 --- a/src/itemdb.d +++ b/src/itemdb.d @@ -388,6 +388,9 @@ final class ItemDatabase } } else { // broken tree + log.vdebug("The following generated a broken tree query:"); + log.vdebug("Drive ID: ", driveId); + log.vdebug("Item ID: ", id); assert(0); } } diff --git a/src/sync.d b/src/sync.d index 27e162e32..12e311e9f 100644 --- a/src/sync.d +++ b/src/sync.d @@ -3100,7 +3100,17 @@ final class SyncEngine string path; // Compute this item path early as we we use this path often - path = itemdb.computePath(item.driveId, item.id); + log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.id); + try { + path = itemdb.computePath(item.driveId, item.id); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + + // item.id was in the database associated with the item.driveId specified log.vlog("Processing ", buildNormalizedPath(path)); // What type of DB item are we processing From 29e611865e4de2fbaf1d35f67a0b22378790405d Mon Sep 17 00:00:00 2001 From: abraunegg Date: Mon, 11 Jan 2021 06:13:19 +1100 Subject: [PATCH 2/3] Update sync.d * Apply same fix for all itemdb.computePath calls --- src/sync.d | 88 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/src/sync.d b/src/sync.d index 12e311e9f..48376393b 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1760,7 +1760,17 @@ final class SyncEngine // What is the original local path for this ID in the database? Does it match 'syncFolderChildPath' if (itemdb.idInLocalDatabase(driveId, item["id"].str)){ // item is in the database - string originalLocalPath = itemdb.computePath(driveId, item["id"].str); + string originalLocalPath; + log.vdebug("Attempting to calculate local filesystem path for ", driveId, " and ", item["id"].str); + try { + originalLocalPath = itemdb.computePath(driveId, item["id"].str); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + if (canFind(originalLocalPath, syncFolderChildPath)){ JSONValue oneDriveMovedNotDeleted; try { @@ -1957,7 +1967,17 @@ final class SyncEngine log.vdebug("skip_dir path to check (simple): ", simplePathToCheck); // complex path if (itemdb.idInLocalDatabase(parentDriveId, parentItem)){ - complexPathToCheck = itemdb.computePath(parentDriveId, parentItem) ~ "/" ~ driveItem["name"].str; + // build up complexPathToCheck + log.vdebug("Attempting to calculate local filesystem path for ", parentDriveId, " and ", parentItem); + try { + complexPathToCheck = itemdb.computePath(parentDriveId, parentItem); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + complexPathToCheck = complexPathToCheck ~ "/" ~ driveItem["name"].str; complexPathToCheck = buildNormalizedPath(complexPathToCheck); } else { log.vdebug("Parent details not in database - unable to compute complex path to check"); @@ -2016,8 +2036,19 @@ final class SyncEngine // is the parent id in the database? if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){ - // need to compute the full path for this file - path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name; + // Compute this item path + log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.parentId); + try { + path = itemdb.computePath(item.driveId, item.parentId); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + + // need the full path for this file + path = path ~ "/" ~ item.name; // The path that needs to be checked needs to include the '/' // This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched @@ -2050,7 +2081,18 @@ final class SyncEngine } else { // Why was this unwanted? if (path.empty) { - path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name; + // Compute this item path + log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.parentId); + try { + path = itemdb.computePath(item.driveId, item.parentId); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + // need the full path for this file + path = path ~ "/" ~ item.name; } // Microsoft OneNote container objects present as neither folder or file but has file size if ((!isItemFile(driveItem)) && (!isItemFolder(driveItem)) && (hasFileSize(driveItem))) { @@ -2070,7 +2112,17 @@ final class SyncEngine // Is the item parent in the local database? if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){ // compute the item path to see if the path is excluded - path = itemdb.computePath(item.driveId, item.parentId) ~ "/" ~ item.name; + log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.parentId); + try { + path = itemdb.computePath(item.driveId, item.parentId); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + // need the full path for this file + path = path ~ "/" ~ item.name; path = buildNormalizedPath(path); if (selectiveSync.isPathExcludedViaSyncList(path)) { // selective sync advised to skip, however is this a file and are we configured to upload / download files in the root? @@ -2165,7 +2217,16 @@ final class SyncEngine // Is the item in the local database if (itemdb.idInLocalDatabase(item.driveId, item.id)){ log.vdebug("OneDrive item ID is present in local database"); - oldPath = itemdb.computePath(item.driveId, item.id); + // Compute this item path + log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.id); + try { + oldPath = itemdb.computePath(item.driveId, item.id); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } // Query DB for existing local item in specified path string itemSource = "database"; if (!isItemSynced(oldItem, oldPath, itemSource)) { @@ -2781,8 +2842,19 @@ final class SyncEngine { foreach_reverse (i; idsToDelete) { Item item; + string path; if (!itemdb.selectById(i[0], i[1], item)) continue; // check if the item is in the db - const(string) path = itemdb.computePath(i[0], i[1]); + // Compute this item path + log.vdebug("Attempting to calculate local filesystem path for ", i[0], " and ", i[1]); + try { + path = itemdb.computePath(i[0], i[1]); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + log.log("Trying to delete item ", path); if (!dryRun) { // Actually process the database entry removal From cbc75b774ed2deb3eb1b9eead1eed6791b746bd9 Mon Sep 17 00:00:00 2001 From: abraunegg Date: Fri, 15 Jan 2021 08:10:36 +1100 Subject: [PATCH 3/3] Update sync.d * code cleanup, remove duplicate code --- src/sync.d | 115 ++++++++++++++--------------------------------------- 1 file changed, 30 insertions(+), 85 deletions(-) diff --git a/src/sync.d b/src/sync.d index 48376393b..ca2f6eb79 100644 --- a/src/sync.d +++ b/src/sync.d @@ -1760,16 +1760,7 @@ final class SyncEngine // What is the original local path for this ID in the database? Does it match 'syncFolderChildPath' if (itemdb.idInLocalDatabase(driveId, item["id"].str)){ // item is in the database - string originalLocalPath; - log.vdebug("Attempting to calculate local filesystem path for ", driveId, " and ", item["id"].str); - try { - originalLocalPath = itemdb.computePath(driveId, item["id"].str); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } + string originalLocalPath = computeItemPath(driveId, item["id"].str); if (canFind(originalLocalPath, syncFolderChildPath)){ JSONValue oneDriveMovedNotDeleted; @@ -1968,16 +1959,7 @@ final class SyncEngine // complex path if (itemdb.idInLocalDatabase(parentDriveId, parentItem)){ // build up complexPathToCheck - log.vdebug("Attempting to calculate local filesystem path for ", parentDriveId, " and ", parentItem); - try { - complexPathToCheck = itemdb.computePath(parentDriveId, parentItem); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } - complexPathToCheck = complexPathToCheck ~ "/" ~ driveItem["name"].str; + complexPathToCheck = computeItemPath(parentDriveId, parentItem) ~ "/" ~ driveItem["name"].str; complexPathToCheck = buildNormalizedPath(complexPathToCheck); } else { log.vdebug("Parent details not in database - unable to compute complex path to check"); @@ -2036,19 +2018,8 @@ final class SyncEngine // is the parent id in the database? if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){ - // Compute this item path - log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.parentId); - try { - path = itemdb.computePath(item.driveId, item.parentId); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } - - // need the full path for this file - path = path ~ "/" ~ item.name; + // Compute this item path & need the full path for this file + path = computeItemPath(item.driveId, item.parentId) ~ "/" ~ item.name; // The path that needs to be checked needs to include the '/' // This due to if the user has specified in skip_file an exclusive path: '/path/file' - that is what must be matched @@ -2081,18 +2052,8 @@ final class SyncEngine } else { // Why was this unwanted? if (path.empty) { - // Compute this item path - log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.parentId); - try { - path = itemdb.computePath(item.driveId, item.parentId); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } - // need the full path for this file - path = path ~ "/" ~ item.name; + // Compute this item path & need the full path for this file + path = computeItemPath(item.driveId, item.parentId) ~ "/" ~ item.name; } // Microsoft OneNote container objects present as neither folder or file but has file size if ((!isItemFile(driveItem)) && (!isItemFolder(driveItem)) && (hasFileSize(driveItem))) { @@ -2111,18 +2072,8 @@ final class SyncEngine if (!unwanted) { // Is the item parent in the local database? if (itemdb.idInLocalDatabase(item.driveId, item.parentId)){ - // compute the item path to see if the path is excluded - log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.parentId); - try { - path = itemdb.computePath(item.driveId, item.parentId); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } - // need the full path for this file - path = path ~ "/" ~ item.name; + // compute the item path to see if the path is excluded & need the full path for this file + path = computeItemPath(item.driveId, item.parentId) ~ "/" ~ item.name; path = buildNormalizedPath(path); if (selectiveSync.isPathExcludedViaSyncList(path)) { // selective sync advised to skip, however is this a file and are we configured to upload / download files in the root? @@ -2218,15 +2169,7 @@ final class SyncEngine if (itemdb.idInLocalDatabase(item.driveId, item.id)){ log.vdebug("OneDrive item ID is present in local database"); // Compute this item path - log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.id); - try { - oldPath = itemdb.computePath(item.driveId, item.id); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } + oldPath = computeItemPath(item.driveId, item.id); // Query DB for existing local item in specified path string itemSource = "database"; if (!isItemSynced(oldItem, oldPath, itemSource)) { @@ -2845,16 +2788,8 @@ final class SyncEngine string path; if (!itemdb.selectById(i[0], i[1], item)) continue; // check if the item is in the db // Compute this item path - log.vdebug("Attempting to calculate local filesystem path for ", i[0], " and ", i[1]); - try { - path = itemdb.computePath(i[0], i[1]); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } - + path = computeItemPath(i[0], i[1]); + // Try to delete item object log.log("Trying to delete item ", path); if (!dryRun) { // Actually process the database entry removal @@ -3172,15 +3107,7 @@ final class SyncEngine string path; // Compute this item path early as we we use this path often - log.vdebug("Attempting to calculate local filesystem path for ", item.driveId, " and ", item.id); - try { - path = itemdb.computePath(item.driveId, item.id); - } catch (core.exception.AssertError) { - // broken tree in the database, we cant compute the path for this item id, exit - log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); - // Must exit here to preserve data - exit(-1); - } + path = computeItemPath(item.driveId, item.id); // item.id was in the database associated with the item.driveId specified log.vlog("Processing ", buildNormalizedPath(path)); @@ -6281,4 +6208,22 @@ final class SyncEngine log.error("ERROR: onedrive.getSharedWithMe call returned an invalid JSON Object"); } } + + // Query itemdb.computePath() and catch potential assert when DB consistency issue occurs + string computeItemPath(string thisDriveId, string thisItemId) + { + string calculatedPath; + log.vdebug("Attempting to calculate local filesystem path for ", thisDriveId, " and ", thisItemId); + try { + calculatedPath = itemdb.computePath(thisDriveId, thisItemId); + } catch (core.exception.AssertError) { + // broken tree in the database, we cant compute the path for this item id, exit + log.error("ERROR: A database consistency issue has been caught. A --resync is needed to rebuild the database."); + // Must exit here to preserve data + exit(-1); + } + + // return calculated path as string + return calculatedPath; + } }