Skip to content

Commit

Permalink
Another Mother of All PRs with the following fixes and tweaks:
Browse files Browse the repository at this point in the history 
- when an SQLite error occurs, a full diagnostics report is included in the logfiles for further diagnosis by the user and/or engineer.
  + this is applied to both local database files (`*.library`)and the 'Sync Point' (`*.s3db`) to help diagnose jimmejardine#257 and similar.
- dialed the Sqlite diagnostics up: Turn on extended result codes via `connection.SetExtendedResultCodes(true);`
- Library integrity checks dialed up several notches: report and track data corruption in the log and pop an alertbox for user to decide to either abort the application or continue as-is -- which should be reasonably safe... - done the same on the Sync Point/Share Target side! .s3db is now integrity-checked on access.
- disambiguated a few confusing log lines.
- quite a bit of work done on metadata record recovery from corrupted/botched/hand-edited SQLite databases (like the ones I still have around from the time of the bad days of Commercial Qiqqa v76-v79, which led to: )
  - some records can be recovered in a way as to treat them as pure BibTeX; adding the fingerprint from the first column as needed.
  - the above is a THIRD option for recovering the metadata and is useful when older databases have been touched by external Sqlite tools: it turns out the previous Qiqqa code did not behave well when a BLOB field was changed in subtle ways by these external tools, including the SQLite CLI itself.
- stop yakking about 'don't know how to yada yada' for libraries which landed in the sync list but do not have a Sync Point: these libraries are now *ignored* (but reported in the log, in case sync was intended and we have a very obscure failure on our hands, e.g. jimmejardine#257 )
- add developer JSON5 option "BuildSearchIndex" to shut up and cut off the Lucene index building/updating background task. This was/is important as we're hitting the very limit of 32-bit memory with Qiqqa while testing with 30+ LARGE old databases in various states of disrepair: at a grand total of 200K+ PDF records, it looks like some re-indexing action will quickly drive us into 'Out Of Memory' fatal failure, while we test for other matters and are NOT interested in the index per se, right now.
- LoadFromMetaData(): add addition sanity check to WARN in log about record key/document fingerprint NOT matching the fingerprint stored in the metadata BLOB. (This should be a RARE error, but indicative of DB data corruption / manual patching and important for diagnosis in your own log files.)
- killed a couple of (even cycling!) 'unhandled exception' dialogs: these buggers are useless when we're already shutting down the app and should therefor only dump the error report in the log.
- correction for the 'end of life of application run' checks and consequent log shutdown.

An overall stability improvement!
  • Loading branch information
@GerHobbelt
GerHobbelt committed Nov 1, 2020
1 parent ee0ae11 commit 4e9bf7d
Show file tree
Hide file tree
Showing 15 changed files with 516 additions and 65 deletions.
7 changes: 7 additions & 0 deletions Qiqqa/Common/BackgroundWorkerDaemonStuff/BackgroundWorkerDaemon.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,12 @@ private void DoMaintenance_Infrequent(Daemon daemon)
return;
}

if (!ConfigurationManager.IsEnabled("BuildSearchIndex"))
{
Logging.Debug特("DoMaintenance_Infrequent::IncrementalBuildIndex: Breaking out of processing loop due to BuildSearchIndex=false");
return;
}

foreach (var web_library_detail in WebLibraryManager.Instance.WebLibraryDetails_WorkingWebLibraries)
{
Library library = web_library_detail.Xlibrary;
Expand Down Expand Up @@ -235,6 +241,7 @@ private void DoMaintenance_Infrequent(Daemon daemon)
Logging.Error(ex, "Exception in metadata_extraction_daemon");
}
}

Logging.Debug特("DoMaintenance_Infrequent END");
}

Expand Down
3 changes: 3 additions & 0 deletions Qiqqa/DocumentLibrary/Import/Auto/MendeleyImporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ internal static MendeleyDatabaseDetails DetectMendeleyDatabaseDetails()
{
using (SQLiteConnection connection = new SQLiteConnection("Data Source=" + sqlite_filename))
{
// Turn on extended result codes
connection.SetExtendedResultCodes(true);

connection.Open();

// Build the authors lookup
Expand Down
85 changes: 78 additions & 7 deletions Qiqqa/DocumentLibrary/IntranetLibraryStuff/IntranetLibraryDB.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Qiqqa.Common.Configuration;
using Utilities;
using Utilities.Files;
using Utilities.GUI;
using Directory = Alphaleonis.Win32.Filesystem.Directory;
using File = Alphaleonis.Win32.Filesystem.File;
using Path = Alphaleonis.Win32.Filesystem.Path;
Expand Down Expand Up @@ -56,7 +57,12 @@ public IntranetLibraryDB(string base_path)

private SQLiteConnection GetConnection()
{
return new SQLiteConnection("Pooling=True;Max Pool Size=3;Data Source=" + library_path);
SQLiteConnection connection = new SQLiteConnection("Pooling=True;Max Pool Size=3;Data Source=" + library_path);

// Turn on extended result codes
connection.SetExtendedResultCodes(true);

return connection;
}

private static readonly char[] queryWildcards = { '*', '?', '%', '_' };
Expand Down Expand Up @@ -170,6 +176,7 @@ public void PutBlob(string filename, byte[] data)
catch (Exception ex)
{
Logging.Error(ex, "IntranetLibraryDB::PutBLOB: Database I/O failure for DB '{0}'.", library_path);
LibraryDB.FurtherDiagnoseDBProblem(ex, null, library_path);
throw ex;
}
}
Expand Down Expand Up @@ -211,6 +218,7 @@ public IntranetLibraryItem GetIntranetLibraryItem(string filename)
public List<IntranetLibraryItem> GetIntranetLibraryItems(string filename, int MaxRecordCount = 0)
{
List<IntranetLibraryItem> results = new List<IntranetLibraryItem>();
List<Exception> database_corruption = new List<Exception>();

try
{
Expand Down Expand Up @@ -243,16 +251,37 @@ public List<IntranetLibraryItem> GetIntranetLibraryItems(string filename, int Ma
IntranetLibraryItem result = new IntranetLibraryItem();
results.Add(result);

long total_bytes = 0;

try
{
result.filename = reader.GetString(0);
result.last_updated_by = reader.GetString(1);
result.md5 = reader.GetString(2);

long total_bytes = reader.GetBytes(3, 0, null, 0, 0);
result.data = new byte[total_bytes];
long total_bytes2 = reader.GetBytes(3, 0, result.data, 0, (int)total_bytes);
if (total_bytes != total_bytes2)
total_bytes = reader.GetBytes(3, 0, null, 0, 0);
result.data = new byte[total_bytes];
long total_bytes2 = reader.GetBytes(3, 0, result.data, 0, (int)total_bytes);
if (total_bytes != total_bytes2)
{
throw new Exception("Error reading blob - blob size different on each occasion.");
}
}
catch (Exception ex)
{
throw new Exception("Error reading blob - blob size different on each occasion.");
string msg = String.Format("IntranetLibraryDB::GetLibraryItems: Database record #{4} decode failure for DB '{0}': filename_id={1}, last_updated_by={2}, md5={3}, length={5}.",
library_path,
String.IsNullOrEmpty(result.filename) ? "???" : result.filename,
String.IsNullOrEmpty(result.last_updated_by) ? "???" : result.last_updated_by,
String.IsNullOrEmpty(result.md5) ? "???" : result.md5,
reader.StepCount, // ~= results.Count + database_corruption.Count
total_bytes
);
Logging.Error(ex, "{0}", msg);

Exception ex2 = new Exception(msg, ex);

database_corruption.Add(ex2);
}
}

Expand Down Expand Up @@ -282,15 +311,28 @@ public List<IntranetLibraryItem> GetIntranetLibraryItems(string filename, int Ma
catch (Exception ex)
{
Logging.Error(ex, "IntranetLibraryDB::GetLibraryItems: Database I/O failure for DB '{0}'.", library_path);
LibraryDB.FurtherDiagnoseDBProblem(ex, database_corruption, library_path);
throw ex;
}

if (database_corruption.Count > 0)
{
// report database corruption: the user may want to recover from this ASAP!
if (MessageBoxes.AskErrorQuestion(true, "INTRANET Library (Sync Point) '{0}' has some data corruption. Do you want to abort the application to attempt recovery using external tools, e.g. a data restore from backup?\n\nWhen you answer NO, we will continue with what we could recover so far instead.\n\n\nConsult the Qiqqa logfiles to see the individual corruptions reported.",
library_path))
{
Logging.Warn("User chose to abort the application on database corruption report");
Environment.Exit(3);
}
}

return results;
}

public List<IntranetLibraryItem> GetIntranetLibraryItemsSummary()
{
List<IntranetLibraryItem> results = new List<IntranetLibraryItem>();
List<Exception> database_corruption = new List<Exception>();

try
{
Expand All @@ -311,8 +353,25 @@ public List<IntranetLibraryItem> GetIntranetLibraryItemsSummary()
IntranetLibraryItem result = new IntranetLibraryItem();
results.Add(result);

try
{
result.filename = reader.GetString(0);
result.md5 = reader.GetString(1);
}
catch (Exception ex)
{
string msg = String.Format("IntranetLibraryDB::GetIntranetLibraryItemsSummary: Database record #{3} decode failure for DB '{0}': filename_id={1}, md5={2}.",
library_path,
String.IsNullOrEmpty(result.filename) ? "???" : result.filename,
String.IsNullOrEmpty(result.md5) ? "???" : result.md5,
reader.StepCount // ~= results.Count + database_corruption.Count
);
Logging.Error(ex, "{0}", msg);

Exception ex2 = new Exception(msg, ex);

database_corruption.Add(ex2);
}
}

reader.Close();
Expand Down Expand Up @@ -340,10 +399,22 @@ public List<IntranetLibraryItem> GetIntranetLibraryItemsSummary()
}
catch (Exception ex)
{
Logging.Error(ex, "IntranetLibraryDB::GetItemsSummary: Database I/O failure for DB '{0}'.", library_path);
Logging.Error(ex, "IntranetLibraryDB::GetLibraryItemsSummary: Database I/O failure for DB '{0}'.", library_path);
LibraryDB.FurtherDiagnoseDBProblem(ex, database_corruption, library_path);
throw ex;
}

if (database_corruption.Count > 0)
{
// report database corruption: the user may want to recover from this ASAP!
if (MessageBoxes.AskErrorQuestion(true, "INTRANET Library (Sync Point) '{0}' has some data corruption. Do you want to abort the application to attempt recovery using external tools, e.g. a data restore from backup?\n\nWhen you answer NO, we will continue with what we could recover so far instead.\n\n\nConsult the Qiqqa logfiles to see the individual corruptions reported.",
library_path))
{
Logging.Warn("User chose to abort the application on database corruption report");
Environment.Exit(3);
}
}

return results;
}
}
Expand Down
2 changes: 1 addition & 1 deletion Qiqqa/DocumentLibrary/IntranetLibraryStuff/IntranetLibraryTools.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ internal static List<string> GetListOfDocumentsInLibrary(string base_path)
Logging.Info("Getting list of documents in Intranet Library {0}", base_path);
string documents_path = GetLibraryDocumentsPath(base_path);
List<string> results = DirectoryTools.GetSubFiles(documents_path, "pdf");
Logging.Info("Got list of documents ({0}) in Intranet Library {1}", results.Count, base_path);
Logging.Info("Got LOCAL list of documents ({0}) in Intranet Library {1}", results.Count, base_path);
return results;
}

Expand Down
16 changes: 9 additions & 7 deletions Qiqqa/DocumentLibrary/Library.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,8 @@ public void BuildFromDocumentRepository(WebLibraryDetail web_library_detail)
long elapsed = 0;
Logging.Debug特("+Build library {0} from repository", web_library_detail.Id);
List<LibraryDB.LibraryItem> library_items = library_db.GetLibraryItems(null, PDFDocumentFileLocations.METADATA);
/* const */ int library_item_count = library_items.Count;
/* const */
int library_item_count = library_items.Count;

elapsed = clk.ElapsedMilliseconds;
Logging.Debug特(":Build library '{2}' from repository -- time spent: {0} ms on fetching {1} records from SQLite DB.", elapsed, library_item_count, web_library_detail.DescriptiveTitle);
Expand All @@ -255,7 +256,8 @@ public void BuildFromDocumentRepository(WebLibraryDetail web_library_detail)
Logging.Info("Library '{2}': Loading {0} files from repository at {1}", library_item_count, web_library_detail.LIBRARY_DOCUMENTS_BASE_PATH, web_library_detail.DescriptiveTitle);

long clk_bound = elapsed;
/* const */ int one_pct_point = library_item_count / 200; // one percent point is defined here as 0.5%
/* const */
int one_pct_point = library_item_count / 200; // one percent point is defined here as 0.5%
int next_i_bound = one_pct_point;
for (int i = 0; i < library_item_count; ++i)
{
Expand Down Expand Up @@ -339,7 +341,7 @@ private void LoadDocumentFromMetadata(LibraryDB.LibraryItem library_item, Dictio
throw new Exception(String.Format("Skipping corrupted NULL record for ID {0}", library_item.ToString()));
}

PDFDocument pdf_document = PDFDocument.LoadFromMetaData(web_library_detail, library_item.data, library_items_annotations_cache);
PDFDocument pdf_document = PDFDocument.LoadFromMetaData(web_library_detail, library_item.fingerprint, library_item.data, library_items_annotations_cache);

//Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_lock)
Expand Down Expand Up @@ -982,7 +984,7 @@ internal void NotifyLibraryThatDocumentListHasChangedExternally()
SignalThatDocumentsHaveChanged(null);
}

#region --- Signaling that documents have been changed ------------------
#region --- Signaling that documents have been changed ------------------

public class PDFDocumentEventArgs : EventArgs
{
Expand Down Expand Up @@ -1068,9 +1070,9 @@ internal void CheckForSignalThatDocumentsHaveChanged()
}
}

#endregion
#endregion

#region --- IDisposable ------------------------------------------------------------------------
#region --- IDisposable ------------------------------------------------------------------------

~Library()
{
Expand Down Expand Up @@ -1143,7 +1145,7 @@ protected virtual void Dispose(bool disposing)
++dispose_count;
}

#endregion
#endregion

}
}

0 comments on commit 4e9bf7d

Please sign in to comment.