v4.0.0

- upgrade to fastJSON v2.2.0 - upgrade to fastBinaryJSON v1.5.1 - changed to faster unicode <-> bytes conversion - optimized view data bytes size without bjson typed arrays - new sparse bitmap index for less memory usage (MGRB based on roaring bitmap) - * possible breaking change if you are using High Frequency key store * - bug fix full text search in columns for case insensitive words - parallel shutdown indexes for better performance - fixed .strings files kept growing
mgholam · Oct 5, 2018 · 93f8694 · 93f8694
1 parent 6caef1f
commit 93f8694
Show file tree

Hide file tree

Showing 10 changed files with 103 additions and 24 deletions.
diff --git a/BuildVersion.cs b/BuildVersion.cs
@@ -1,6 +1,6 @@
 using System.Reflection;
-// build number = 511
+// build number = 515
 // build version = 4.0.0
 
 [assembly: AssemblyVersion("4.0.0.0")]
-[assembly: AssemblyFileVersion("4.0.0.511")]
+[assembly: AssemblyFileVersion("4.0.0.515")]
diff --git a/RaptorDB.Common/SafeDictionary.cs b/RaptorDB.Common/SafeDictionary.cs
@@ -335,12 +335,12 @@ public static unsafe byte[] GetBytes(short num, bool reverse)
 
         public static byte[] GetBytes(string s)
         {
-            return Encoding.UTF8.GetBytes(s); // FIX : change to unicode ??
+            return Encoding.UTF8.GetBytes(s); // TODO : change to unicode ??
         }
 
         public static string GetString(byte[] buffer, int index, short length)
         {
-            return Encoding.UTF8.GetString(buffer, index, length); // FIX : change to unicode ??
+            return Encoding.UTF8.GetString(buffer, index, length); // TODO : change to unicode ??
         }
     }
 }
diff --git a/RaptorDB/Helper/Container.cs b/RaptorDB/Helper/Container.cs
@@ -498,10 +498,10 @@ public static int BitCount(ulong x)
 
     public enum CTYPE
     {
-        ALLONES,
-        BITMAP,
-        OFFSET,
-        OFFSETSL
+        ALLONES
+        ,BITMAP
+        ,OFFSET
+        //,OFFSETSL
     }
     public class CData
     {

diff --git a/RaptorDB/Helper/MGRB.cs b/RaptorDB/Helper/MGRB.cs
@@ -48,8 +48,9 @@ public void Set(long position, bool val)
                     _containers.Add(idx, c);
                 }
                 c.Set(position & _MASK, val);
-                if (c.ChangeRequired())
-                    _containers[idx] = c.Change();
+
+                //if (c.ChangeRequired())
+                //    _containers[idx] = c.Change();
             }
         }
 

diff --git a/RaptorDB/Indexes/MGIndex.cs b/RaptorDB/Indexes/MGIndex.cs
@@ -285,7 +285,6 @@ public void FreeMemory()
             catch { }
         }
 
-
         public IEnumerable<int> GetDuplicates(T key)
         {
             PageInfo pi;

diff --git a/RaptorDB/RaptorDB.cs b/RaptorDB/RaptorDB.cs
@@ -1,16 +1,17 @@
-using System;
+using RaptorDB.Common;
+using RaptorDB.Views;
+using System;
+using System.CodeDom.Compiler;
 using System.Collections.Generic;
-using System.Text;
+using System.ComponentModel;
 using System.IO;
-using System.Threading;
-using RaptorDB.Views;
+using System.IO.Compression;
+using System.Linq;
 using System.Linq.Expressions;
 using System.Reflection;
-using RaptorDB.Common;
-using System.IO.Compression;
-using System.CodeDom.Compiler;
+using System.Text;
 using System.Text.RegularExpressions;
-using System.ComponentModel;
+using System.Threading;
 
 // ----- Feature list -------
 // TODO : enum in row schema support
@@ -89,6 +90,8 @@ public static RaptorDB Open(string FolderPath, ITokenizer tokenizer)
         private Replication.ReplicationClient _repclient;
         private DateTime _startTime = DateTime.Now;
         private ITokenizer _tokenizer = new tokenizer();
+        private int _RaptorDBVersion = 4;
+
 
         #region [            P U B L I C    I N T E R F A C E            ]
         /// <summary>
@@ -1123,6 +1126,11 @@ private void Initialize()
             if (v < StorageFile<int>._CurrentVersion)
                 UpgradeStorageFile(_Path + "Data" + _S + "files", v);
 
+            // upgrade old mgidx files 
+            if (File.Exists(_Path + "Data" + _S + "RaptorDB.version") == false)
+            {
+                RebuildDataFiles();
+            }
             _objStore = new KeyStore<Guid>(_Path + "Data" + _S + "data", true);
             _fileStore = new KeyStore<Guid>(_Path + "Data" + _S + "files", true);
 
@@ -1209,6 +1217,68 @@ private void Initialize()
             }
         }
 
+        private void RebuildDataFiles()
+        {
+            try
+            {
+                var data = _Path + "data" + _S;
+                var tmp = data + "temp" + _S;
+                _log.Debug("Rebuilding MGDAT files...");
+
+                _log.Debug("  deleting Fulltext folder.");
+                // delete "fulltext" folder -> will rebuild automatically
+                Directory.Delete(data + "Fulltext", true);
+
+                _log.Debug("  moving mgdat files to temp folder.");
+                Directory.CreateDirectory(tmp);
+                File.Move(data + "data.mgdat", tmp + "data.mgdat");
+                File.Move(data + "files.mgdat", tmp + "files.mgdat");
+
+                // delete all index files -> keep data.mgdat & files.mgdat
+                Directory.GetFiles(data, "*.*").ToList().ForEach(x => File.Delete(x));
+
+                // upgrade by rebuilding indexes 
+                var sf = StorageFile<Guid>.ReadForward(tmp + "data.mgdat");
+                var obstore = new KeyStore<Guid>(data + "data", true);
+                _log.Debug("  rebuilding index for data.mgdat");
+                _log.Debug("  docs count = " + sf.Count());
+                foreach (var i in sf.ReadOnlyEnumerate())
+                {
+                    if (i.meta.isDeleted)
+                        obstore.Delete(i.meta.key);
+                    else
+                    {
+                        var o = CreateObject(i.data);
+                        obstore.SetObject(i.meta.key, o);
+                    }
+                }
+                obstore.Shutdown();
+                sf.Shutdown();
+                sf = StorageFile<Guid>.ReadForward(tmp + "files.mgdat");
+                obstore = new KeyStore<Guid>(data + "files", true);
+                _log.Debug("  rebuilding index for files.mgdat");
+                _log.Debug("  files count = " + sf.Count());
+                foreach (var i in sf.ReadOnlyEnumerate())
+                {
+                    if (i.meta.isDeleted)
+                        obstore.Delete(i.meta.key);
+                    else
+                    {
+                        var o = CreateObject(i.data);
+                        obstore.SetObject(i.meta.key, o);
+                    }
+                }
+                obstore.Shutdown();
+                sf.Shutdown();
+                File.WriteAllText(data + "RaptorDB.version", "" + _RaptorDBVersion);
+                Directory.Delete(tmp, true);
+            }
+            catch (Exception ex)
+            {
+                _log.Error(ex);
+            }
+        }
+
         object _inboxlock = new object();
         void _processinboxTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
         {

diff --git a/RaptorDB/Storage/KeyStoreHF.cs b/RaptorDB/Storage/KeyStoreHF.cs
@@ -443,7 +443,7 @@ private void RebuildDataFiles()
 
                 int c = _datastore.NumberofBlocks();
 
-                for (int i = 0; i < c; i++) // go through blocks
+                for (int i = 1; i < c; i++) // go through blocks skip first
                 {
                     if (visited.Get(i))
                         continue;

diff --git a/datagridbinding/frmMain.Designer.cs b/datagridbinding/frmMain.Designer.cs
diff --git a/datagridbinding/frmMain.cs b/datagridbinding/frmMain.cs
@@ -320,6 +320,15 @@ private void freememoryToolStripMenuItem_Click(object sender, EventArgs e)
             rap.FreeMemory();
         }
 
+        private void dataGridView1_CellDoubleClick(object sender, DataGridViewCellEventArgs e)
+        {
+            var id = (Guid)dataGridView1.Rows[e.RowIndex].Cells["docid"].Value;
+
+            var s = fastJSON.JSON.ToNiceJSON( rap.Fetch(id) , new fastJSON.JSONParameters {UseExtensions= false, UseFastGuid = false });
+
+            MessageBox.Show(s, "JSON Value");
+        }
+
         //private void perftest()
         //{
         //    DateTime dt = DateTime.Now;

diff --git a/history.txt b/history.txt
@@ -1,15 +1,14 @@
-v4.0.0*
+v4.0.0
 -------
 - upgrade to fastJSON v2.2.0
 - upgrade to fastBinaryJSON v1.5.1
 - changed to faster unicode <-> bytes conversion
-- optimized view data size without bjson typed arrays
+- optimized view data bytes size without bjson typed arrays
 - new sparse bitmap index for less memory usage (MGRB based on roaring bitmap)
 - * possible breaking change if you are using High Frequency key store *
 - bug fix full text search in columns for case insensitive words
-- parallel shutdown indexes 
+- parallel shutdown indexes for better performance
 - fixed .strings files kept growing
-- 
 
 v3.3.19.1
 ---------