Skip to content

Commit

Permalink
- FreeMemory() in IRaptorDB
Browse files Browse the repository at this point in the history
  • Loading branch information
@mgholam
mgholam committed Oct 4, 2018
1 parent 204e279 commit 6caef1f
Show file tree
Hide file tree
Showing 14 changed files with 98 additions and 58 deletions.
4 changes: 2 additions & 2 deletions BuildVersion.cs
View file
@@ -1,6 +1,6 @@
using System.Reflection;
// build number = 508
// build number = 511
// build version = 4.0.0

[assembly: AssemblyVersion("4.0.0.0")]
[assembly: AssemblyFileVersion("4.0.0.508")]
[assembly: AssemblyFileVersion("4.0.0.511")]
2 changes: 2 additions & 0 deletions RaptorDB.Common/IRaptorDB.cs
View file
Expand Up @@ -369,6 +369,8 @@ public interface IRaptorDB
/// </summary>
IKeyStoreHF GetKVHF();

void FreeMemory();

void Shutdown();
}
}
3 changes: 2 additions & 1 deletion RaptorDB.Common/Interfaces.cs
View file
Expand Up @@ -194,6 +194,7 @@ public enum COMMANDS
CompactStorageHF,
IncrementHF,
DecrementHF,
ServerSideWithArgs
ServerSideWithArgs,
FreeMemory
}
}
7 changes: 7 additions & 0 deletions RaptorDB.Common/RaptorDBClient.cs
View file
Expand Up @@ -863,5 +863,12 @@ public object[] ServerSide<TRowSchema>(ServerSideFuncWithArgs func, Expression<P
else
return null;
}

public void FreeMemory()
{
Packet p = CreatePacket();
p.Command = "" + COMMANDS.FreeMemory;
ReturnPacket ret = (ReturnPacket)_client.Send(p);
}
}
}
22 changes: 11 additions & 11 deletions RaptorDB/Indexes/BitmapIndex.cs
View file
Expand Up @@ -159,7 +159,7 @@ public void Optimize()
{
long offset = ReadRecordOffset(i);

byte[] b = ReadBMPData(offset);
byte[] b = ReadBMPDataForOptimize(offset);
if (b == null)
{
_stopOperations = false;
Expand Down Expand Up @@ -358,7 +358,7 @@ private long SaveBitmapToFile(MGRB bmp)
hdr[1] = (byte)'m';
hdr[2] = 0; // uncompressed

if(Global.CompressBitmapBytes)
if (Global.CompressBitmapBytes)
{
hdr[2] = 1;
b = MiniLZO.Compress(b);
Expand All @@ -376,20 +376,20 @@ private long SaveBitmapToFile(MGRB bmp)
return off;
}

private byte[] ReadBMPData(long offset)
private byte[] ReadBMPDataForOptimize(long offset)
{
// fix : fix this code to new format
// return data + header
_bitmapFileRead.Seek(offset, SeekOrigin.Begin);

byte[] b = new byte[8];
byte[] hdr = new byte[_hdrlen];

_bitmapFileRead.Read(b, 0, 8);
if (b[0] == (byte)'b' && b[1] == (byte)'m')// && b[7] == 0)
_bitmapFileRead.Read(hdr, 0, _hdrlen);
if (hdr[0] == (byte)'b' && hdr[1] == (byte)'m')
{
int c = Helper.ToInt32(b, 2) * 4 + 8;
byte[] data = new byte[c];
_bitmapFileRead.Seek(offset, SeekOrigin.Begin);
_bitmapFileRead.Read(data, 0, c);
int c = Helper.ToInt32(hdr, 3);
var data = new byte[c + _hdrlen];
Buffer.BlockCopy(hdr, 0, data, 0, _hdrlen);
_bitmapFileRead.Read(data, _hdrlen, c);
return data;
}
return null;
Expand Down
36 changes: 19 additions & 17 deletions RaptorDB/Indexes/IndexFile.cs
View file
Expand Up @@ -40,7 +40,7 @@ internal class IndexFile<T>

private KeyStoreHF _strings;
private bool _externalStrings = false;
private List<int> _pagelistalllocblock = null;
//private List<int> _pagelistalllocblock = null;
private string _FileName = "";

public IndexFile(string filename, byte maxKeySize)//, ushort pageNodeCount)
Expand Down Expand Up @@ -267,13 +267,14 @@ private int LoadPageListData(int page, SafeSortedList<T, PageInfo> PageList)
nextpage = Helper.ToInt32(b, 11);
int index = _BlockHeader.Length;
object[] keys = null;
if (File.Exists(_FileName + ".pagelist"))
{
var bn = File.ReadAllBytes(_FileName + ".pagelist");
int blknum = Helper.ToInt32(bn, 0);
byte[] bb = _strings.GetData(blknum, _pagelistalllocblock);
keys = (object[])BJSON.ToObject(bb);
}
// TODO : needed??
//if (File.Exists(_FileName + ".pagelist"))
//{
// var bn = File.ReadAllBytes(_FileName + ".pagelist");
// int blknum = Helper.ToInt32(bn, 0);
// byte[] bb = _strings.GetData(blknum, out _pagelistalllocblock);
// keys = (object[])BJSON.ToObject(bb);
//}
for (int i = 0; i < count; i++)
{
int idx = index + _rowSize * i;
Expand Down Expand Up @@ -398,7 +399,7 @@ public Page<T> LoadPageFromPageNumber(int number)
if (keys == null)
{
int blknum = Helper.ToInt32(b, idx + 1, false);
byte[] bb = _strings.GetData(blknum, page.allocblocks);
byte[] bb = _strings.GetData(blknum, out page.allocblocks);
keys = (object[])BJSON.ToObject(bb);
}
key = (T)keys[i];
Expand All @@ -421,14 +422,15 @@ internal void SavePageList(SafeSortedList<T, PageInfo> _pages, List<int> diskpag
{
T[] keys = _pages.Keys();
int blocknum = 0;
if (_externalStrings)
{
if (_pagelistalllocblock != null)
_strings.FreeBlocks(_pagelistalllocblock);
blocknum = _strings.SaveData("pagelist", BJSON.ToBJSON(keys,
new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false }));
File.WriteAllBytes(_FileName + ".pagelist", Helper.GetBytes(blocknum, false));
}
// TODO : needed??
//if (_externalStrings)
//{
// if (_pagelistalllocblock != null)
// _strings.FreeBlocks(_pagelistalllocblock);
// blocknum = _strings.SaveData("pagelist", BJSON.ToBJSON(keys,
// new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false }));
// File.WriteAllBytes(_FileName + ".pagelist", Helper.GetBytes(blocknum, false));
//}
// save page list
int c = (_pages.Count() / Global.PageItemCount) + 1;
// allocate pages needed
Expand Down
29 changes: 17 additions & 12 deletions RaptorDB/RaptorDB.cs
View file
Expand Up @@ -1323,18 +1323,7 @@ private Assembly CompileScript(string file)

void _freeMemTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
long l = GC.GetTotalMemory(true) / (1024 * 1024);
_log.Debug("GC.GetTotalMemory() = " + l.ToString("#,0"));
if (l > Global.MemoryLimit)
{
_log.Debug("Freeing memory on " + Global.MemoryLimit.ToString("#,0") + " limit ...");
_viewManager.FreeMemory();
_fulltextindex.FreeMemory();
_objStore.FreeMemory();
_fileStore.FreeMemory();
_objHF.FreeMemory();
GC.Collect();// GC.MaxGeneration);
}
FreeMemory();
}

private void UpgradeStorageFile(string filename, int ver)
Expand Down Expand Up @@ -1513,6 +1502,22 @@ internal object GetAssemblyForView(string viewname, out string typename)
return Fetch(docID) as T;
}

public void FreeMemory()
{
long l = GC.GetTotalMemory(true) / (1024 * 1024);
_log.Debug("GC.GetTotalMemory() = " + l.ToString("#,0"));
if (l > Global.MemoryLimit)
{
_log.Debug("Freeing memory on " + Global.MemoryLimit.ToString("#,0") + " limit ...");
_viewManager.FreeMemory();
_fulltextindex.FreeMemory();
_objStore.FreeMemory();
_fileStore.FreeMemory();
_objHF.FreeMemory();
GC.Collect();// GC.MaxGeneration);
}
}

#endregion
}
}
8 changes: 8 additions & 0 deletions RaptorDB/RaptorDBServer.cs
View file
Expand Up @@ -549,6 +549,14 @@ private void InitializeCommandsDictionary()
ret.OK = true;
ret.Data = _defaultInstance.rdb.ServerSide(GetServerSideFuncWithArgsCache(param[0].ToString(), param[1].ToString()), param[2].ToString(), param[3]);
});

_handlers.Add("" + COMMANDS.FreeMemory,
(p, ret) =>
{
ret.OK = true;
_log.Debug("Free memory called from client");
_defaultInstance.rdb.FreeMemory();
});
}

private ServerSideFuncWithArgs GetServerSideFuncWithArgsCache(string type, string method)
Expand Down
2 changes: 0 additions & 2 deletions RaptorDB/Storage/KeyStore.cs
View file
Expand Up @@ -477,8 +477,6 @@ private void Initialize(string filename, byte maxkeysize, bool AllowDuplicateKey

//LogManager.Configure(_Path + Path.DirectorySeparatorChar + _FileName + ".txt", 500, false);

// FIX : check if indexes need upgrade to mgrb

_index = new MGIndex<T>(_Path, _FileName + _idxExtension, _MaxKeySize, /*Global.PageItemCount,*/ AllowDuplicateKeys);

if (Global.SaveAsBinaryJSON)
Expand Down
4 changes: 2 additions & 2 deletions RaptorDB/Storage/KeyStoreHF.cs
View file
Expand Up @@ -214,7 +214,7 @@ public void CompactStorageHF()
public string[] GetKeysHF()
{
lock (_lock)
return _keys.GetKeys().Cast<string>().ToArray(); // FEATURE : dirty !?
return _keys.GetKeys().Cast<string>().ToArray(); // FEATURE : ugly and dirty !?
}

public bool ContainsHF(string key)
Expand Down Expand Up @@ -549,7 +549,7 @@ internal int SaveData(string key, byte[] data)
}

// for .string files
internal byte[] GetData(int blocknumber, List<int> usedblocks)
internal byte[] GetData(int blocknumber, out List<int> usedblocks)
{
lock (_lock)
{
Expand Down
15 changes: 8 additions & 7 deletions RaptorDB/Storage/StorageFileHF.cs
View file
Expand Up @@ -108,12 +108,11 @@ internal void Initialize()
// get free block num and size
int block = Helper.ToInt32(b, 2);
int len = Helper.ToInt32(b, 2 + 4);
// read blocks upto size from block num
// read freelist from master block from end of file
_lastBlockNumber = block;
b = new byte[len];
SeekBlock(block);
var offset = 0;
// read blocks upto size from block num
SeekBlock(block);
while (len > 0)
{
// check header
Expand All @@ -127,7 +126,9 @@ internal void Initialize()
int c = len > _BLOCKSIZE ? _BLOCKSIZE - 2 : len;
Buffer.BlockCopy(bb, 2, b, offset, c);
len -= c;
offset += c;
}
// read freelist from master block from end of file
var o = fastBinaryJSON.BJSON.ToObject<MGRBData>(b);
_freeList.Deserialize(o);
}
Expand All @@ -137,7 +138,8 @@ internal void Initialize()
internal void SeekBlock(int blocknumber)
{
long offset = _fileheader.Length + (long)blocknumber * _BLOCKSIZE;
_datawrite.Seek(offset, SeekOrigin.Begin);// wiil seek past the end of file on fs.Write will zero the difference
// wiil seek past the end of file on fs.Write will zero the difference
_datawrite.Seek(offset, SeekOrigin.Begin);
}

internal void WriteBlockBytes(byte[] data, int start, int len)
Expand All @@ -150,7 +152,6 @@ internal void WriteBlockBytes(byte[] data, int start, int len)
private void WriteFreeListBMPFile()
{
// write freelist to end of blocks and update master block

if (_freeList != null)
{
_freeList.Optimize();
Expand Down Expand Up @@ -214,7 +215,7 @@ private int ReadFileHeader()
_datawrite.Read(hdr, 0, _fileheader.Length);

_BLOCKSIZE = 0;
_BLOCKSIZE = (ushort)((int)hdr[5] + ((int)hdr[6]) << 8);
_BLOCKSIZE = (ushort)(hdr[5] + hdr[6] << 8);

return hdr[4];
}
Expand All @@ -241,7 +242,7 @@ private void FlushClose(FileStream st)

internal int NumberofBlocks()
{
return (int)((_datawrite.Length / (int)_BLOCKSIZE)) + 1;
return (int)((_datawrite.Length / _BLOCKSIZE)) + 1;
}

internal void FreeBlock(int i)
Expand Down
18 changes: 14 additions & 4 deletions datagridbinding/frmMain.Designer.cs
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions datagridbinding/frmMain.cs
View file
Expand Up @@ -315,6 +315,11 @@ private void frmMain_FormClosing(object sender, FormClosingEventArgs e)
shutdown();
}

private void freememoryToolStripMenuItem_Click(object sender, EventArgs e)
{
rap.FreeMemory();
}

//private void perftest()
//{
// DateTime dt = DateTime.Now;
Expand Down
1 change: 1 addition & 0 deletions history.txt
View file
Expand Up @@ -8,6 +8,7 @@
- * possible breaking change if you are using High Frequency key store *
- bug fix full text search in columns for case insensitive words
- parallel shutdown indexes
- fixed .strings files kept growing
-

v3.3.19.1
Expand Down

0 comments on commit 6caef1f

Please sign in to comment.