From 93f8694d3bbfa4b51d7c17f6beee01de277bcf49 Mon Sep 17 00:00:00 2001 From: "M. Gholam" Date: Fri, 5 Oct 2018 11:48:56 +0330 Subject: [PATCH] v4.0.0 - upgrade to fastJSON v2.2.0 - upgrade to fastBinaryJSON v1.5.1 - changed to faster unicode <-> bytes conversion - optimized view data bytes size without bjson typed arrays - new sparse bitmap index for less memory usage (MGRB based on roaring bitmap) - * possible breaking change if you are using High Frequency key store * - bug fix full text search in columns for case insensitive words - parallel shutdown indexes for better performance - fixed .strings files kept growing --- BuildVersion.cs | 4 +- RaptorDB.Common/SafeDictionary.cs | 4 +- RaptorDB/Helper/Container.cs | 8 +-- RaptorDB/Helper/MGRB.cs | 5 +- RaptorDB/Indexes/MGIndex.cs | 1 - RaptorDB/RaptorDB.cs | 86 ++++++++++++++++++++++++++--- RaptorDB/Storage/KeyStoreHF.cs | 2 +- datagridbinding/frmMain.Designer.cs | 1 + datagridbinding/frmMain.cs | 9 +++ history.txt | 7 +-- 10 files changed, 103 insertions(+), 24 deletions(-) diff --git a/BuildVersion.cs b/BuildVersion.cs index a93d467..7e3bbe3 100644 --- a/BuildVersion.cs +++ b/BuildVersion.cs @@ -1,6 +1,6 @@ using System.Reflection; -// build number = 511 +// build number = 515 // build version = 4.0.0 [assembly: AssemblyVersion("4.0.0.0")] -[assembly: AssemblyFileVersion("4.0.0.511")] \ No newline at end of file +[assembly: AssemblyFileVersion("4.0.0.515")] \ No newline at end of file diff --git a/RaptorDB.Common/SafeDictionary.cs b/RaptorDB.Common/SafeDictionary.cs index 91f4dc2..f4c52da 100644 --- a/RaptorDB.Common/SafeDictionary.cs +++ b/RaptorDB.Common/SafeDictionary.cs @@ -335,12 +335,12 @@ public static unsafe byte[] GetBytes(short num, bool reverse) public static byte[] GetBytes(string s) { - return Encoding.UTF8.GetBytes(s); // FIX : change to unicode ?? + return Encoding.UTF8.GetBytes(s); // TODO : change to unicode ?? } public static string GetString(byte[] buffer, int index, short length) { - return Encoding.UTF8.GetString(buffer, index, length); // FIX : change to unicode ?? + return Encoding.UTF8.GetString(buffer, index, length); // TODO : change to unicode ?? } } } diff --git a/RaptorDB/Helper/Container.cs b/RaptorDB/Helper/Container.cs index 109d294..f14fbfc 100644 --- a/RaptorDB/Helper/Container.cs +++ b/RaptorDB/Helper/Container.cs @@ -498,10 +498,10 @@ public static int BitCount(ulong x) public enum CTYPE { - ALLONES, - BITMAP, - OFFSET, - OFFSETSL + ALLONES + ,BITMAP + ,OFFSET + //,OFFSETSL } public class CData { diff --git a/RaptorDB/Helper/MGRB.cs b/RaptorDB/Helper/MGRB.cs index 64a094d..27ace0e 100644 --- a/RaptorDB/Helper/MGRB.cs +++ b/RaptorDB/Helper/MGRB.cs @@ -48,8 +48,9 @@ public void Set(long position, bool val) _containers.Add(idx, c); } c.Set(position & _MASK, val); - if (c.ChangeRequired()) - _containers[idx] = c.Change(); + + //if (c.ChangeRequired()) + // _containers[idx] = c.Change(); } } diff --git a/RaptorDB/Indexes/MGIndex.cs b/RaptorDB/Indexes/MGIndex.cs index c00dd07..656a9af 100644 --- a/RaptorDB/Indexes/MGIndex.cs +++ b/RaptorDB/Indexes/MGIndex.cs @@ -285,7 +285,6 @@ public void FreeMemory() catch { } } - public IEnumerable GetDuplicates(T key) { PageInfo pi; diff --git a/RaptorDB/RaptorDB.cs b/RaptorDB/RaptorDB.cs index a330cec..ee7a21b 100644 --- a/RaptorDB/RaptorDB.cs +++ b/RaptorDB/RaptorDB.cs @@ -1,16 +1,17 @@ -using System; +using RaptorDB.Common; +using RaptorDB.Views; +using System; +using System.CodeDom.Compiler; using System.Collections.Generic; -using System.Text; +using System.ComponentModel; using System.IO; -using System.Threading; -using RaptorDB.Views; +using System.IO.Compression; +using System.Linq; using System.Linq.Expressions; using System.Reflection; -using RaptorDB.Common; -using System.IO.Compression; -using System.CodeDom.Compiler; +using System.Text; using System.Text.RegularExpressions; -using System.ComponentModel; +using System.Threading; // ----- Feature list ------- // TODO : enum in row schema support @@ -89,6 +90,8 @@ public static RaptorDB Open(string FolderPath, ITokenizer tokenizer) private Replication.ReplicationClient _repclient; private DateTime _startTime = DateTime.Now; private ITokenizer _tokenizer = new tokenizer(); + private int _RaptorDBVersion = 4; + #region [ P U B L I C I N T E R F A C E ] /// @@ -1123,6 +1126,11 @@ private void Initialize() if (v < StorageFile._CurrentVersion) UpgradeStorageFile(_Path + "Data" + _S + "files", v); + // upgrade old mgidx files + if (File.Exists(_Path + "Data" + _S + "RaptorDB.version") == false) + { + RebuildDataFiles(); + } _objStore = new KeyStore(_Path + "Data" + _S + "data", true); _fileStore = new KeyStore(_Path + "Data" + _S + "files", true); @@ -1209,6 +1217,68 @@ private void Initialize() } } + private void RebuildDataFiles() + { + try + { + var data = _Path + "data" + _S; + var tmp = data + "temp" + _S; + _log.Debug("Rebuilding MGDAT files..."); + + _log.Debug(" deleting Fulltext folder."); + // delete "fulltext" folder -> will rebuild automatically + Directory.Delete(data + "Fulltext", true); + + _log.Debug(" moving mgdat files to temp folder."); + Directory.CreateDirectory(tmp); + File.Move(data + "data.mgdat", tmp + "data.mgdat"); + File.Move(data + "files.mgdat", tmp + "files.mgdat"); + + // delete all index files -> keep data.mgdat & files.mgdat + Directory.GetFiles(data, "*.*").ToList().ForEach(x => File.Delete(x)); + + // upgrade by rebuilding indexes + var sf = StorageFile.ReadForward(tmp + "data.mgdat"); + var obstore = new KeyStore(data + "data", true); + _log.Debug(" rebuilding index for data.mgdat"); + _log.Debug(" docs count = " + sf.Count()); + foreach (var i in sf.ReadOnlyEnumerate()) + { + if (i.meta.isDeleted) + obstore.Delete(i.meta.key); + else + { + var o = CreateObject(i.data); + obstore.SetObject(i.meta.key, o); + } + } + obstore.Shutdown(); + sf.Shutdown(); + sf = StorageFile.ReadForward(tmp + "files.mgdat"); + obstore = new KeyStore(data + "files", true); + _log.Debug(" rebuilding index for files.mgdat"); + _log.Debug(" files count = " + sf.Count()); + foreach (var i in sf.ReadOnlyEnumerate()) + { + if (i.meta.isDeleted) + obstore.Delete(i.meta.key); + else + { + var o = CreateObject(i.data); + obstore.SetObject(i.meta.key, o); + } + } + obstore.Shutdown(); + sf.Shutdown(); + File.WriteAllText(data + "RaptorDB.version", "" + _RaptorDBVersion); + Directory.Delete(tmp, true); + } + catch (Exception ex) + { + _log.Error(ex); + } + } + object _inboxlock = new object(); void _processinboxTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { diff --git a/RaptorDB/Storage/KeyStoreHF.cs b/RaptorDB/Storage/KeyStoreHF.cs index df24fab..a2b9af7 100644 --- a/RaptorDB/Storage/KeyStoreHF.cs +++ b/RaptorDB/Storage/KeyStoreHF.cs @@ -443,7 +443,7 @@ private void RebuildDataFiles() int c = _datastore.NumberofBlocks(); - for (int i = 0; i < c; i++) // go through blocks + for (int i = 1; i < c; i++) // go through blocks skip first { if (visited.Get(i)) continue; diff --git a/datagridbinding/frmMain.Designer.cs b/datagridbinding/frmMain.Designer.cs index 4d0ad70..4f41a6c 100644 --- a/datagridbinding/frmMain.Designer.cs +++ b/datagridbinding/frmMain.Designer.cs @@ -182,6 +182,7 @@ private void InitializeComponent() this.dataGridView1.Name = "dataGridView1"; this.dataGridView1.Size = new System.Drawing.Size(693, 317); this.dataGridView1.TabIndex = 7; + this.dataGridView1.CellDoubleClick += new System.Windows.Forms.DataGridViewCellEventHandler(this.dataGridView1_CellDoubleClick); // // freememoryToolStripMenuItem // diff --git a/datagridbinding/frmMain.cs b/datagridbinding/frmMain.cs index 022f85c..162737c 100644 --- a/datagridbinding/frmMain.cs +++ b/datagridbinding/frmMain.cs @@ -320,6 +320,15 @@ private void freememoryToolStripMenuItem_Click(object sender, EventArgs e) rap.FreeMemory(); } + private void dataGridView1_CellDoubleClick(object sender, DataGridViewCellEventArgs e) + { + var id = (Guid)dataGridView1.Rows[e.RowIndex].Cells["docid"].Value; + + var s = fastJSON.JSON.ToNiceJSON( rap.Fetch(id) , new fastJSON.JSONParameters {UseExtensions= false, UseFastGuid = false }); + + MessageBox.Show(s, "JSON Value"); + } + //private void perftest() //{ // DateTime dt = DateTime.Now; diff --git a/history.txt b/history.txt index de4e8db..63e5607 100644 --- a/history.txt +++ b/history.txt @@ -1,15 +1,14 @@ -v4.0.0* +v4.0.0 ------- - upgrade to fastJSON v2.2.0 - upgrade to fastBinaryJSON v1.5.1 - changed to faster unicode <-> bytes conversion -- optimized view data size without bjson typed arrays +- optimized view data bytes size without bjson typed arrays - new sparse bitmap index for less memory usage (MGRB based on roaring bitmap) - * possible breaking change if you are using High Frequency key store * - bug fix full text search in columns for case insensitive words -- parallel shutdown indexes +- parallel shutdown indexes for better performance - fixed .strings files kept growing -- v3.3.19.1 ---------