Skip to content

Commit

Permalink
work with large files (~15GB tested)
Browse files Browse the repository at this point in the history
  • Loading branch information
@b4tman
b4tman committed Aug 11, 2021
1 parent a41965b commit c13311e
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 26 deletions.
34 changes: 20 additions & 14 deletions PasswordChanger1C/AccessFunctions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ public static class AccessFunctions
{
public struct StorageTable
{
public int Number;
public List<int> DataBlocks;
public long Number;
public List<long> DataBlocks;
}

public struct TableFields
Expand All @@ -20,7 +20,7 @@ public struct TableFields
public int Length;
public int Precision;
public int Size;
public int Offset;
public long Offset;
public string Type;
public int CouldBeNull;
}
Expand All @@ -34,13 +34,13 @@ public struct PageParams
public int version1;
public int version2;
public int version;
public List<int> PagesNum;
public List<long> PagesNum;
public List<StorageTable> StorageTables;
public List<TableFields> Fields;
public int RowSize;
public long RowSize;
public List<Dictionary<string, object>> Records;
public int BlockData;
public int BlockBlob;
public long BlockData;
public long BlockBlob;
public int PageSize;
public string TableDefinition;
public string DatabaseVersion;
Expand Down Expand Up @@ -89,7 +89,7 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag
{
byte[] bytesBlock;
PageParams DataPage;
int TotalBlocks;
long TotalBlocks;
int i;

using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write))
Expand All @@ -100,7 +100,7 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag
reader.BaseStream.Seek(PageHeader.BlockData * 4096, SeekOrigin.Begin);
reader.Read(bytesBlock1, 0, 4096);
DataPage = DatabaseAccess8214.ReadPage(reader, bytesBlock1);
TotalBlocks = DataPage.StorageTables.Sum(ST => ST.DataBlocks.Count);
TotalBlocks = DataPage.StorageTables.Sum(ST => (long)ST.DataBlocks.Count);
bytesBlock = new byte[4096 * TotalBlocks];
i = 0;
foreach (var ST in DataPage.StorageTables)
Expand All @@ -123,6 +123,7 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag
using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write))
{
using var writer = new BinaryWriter(fs);
long LastPos = 0;
i = 0;
foreach (var ST in DataPage.StorageTables)
{
Expand All @@ -132,7 +133,9 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag
bytesBlock.AsMemory(i, TempBlock.Length).CopyTo(TempBlock.AsMemory());
i += TempBlock.Length;

writer.Seek(DB * 4096, SeekOrigin.Begin);
long CurPos = DB * 4096;
int RelativePos = (int)(CurPos - LastPos);
writer.Seek(RelativePos, SeekOrigin.Current);
writer.Write(TempBlock);
}
}
Expand All @@ -148,7 +151,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH
}

int i;
int TotalBlocks;
long TotalBlocks;
byte[] bytesBlock;
int PageSize = PageHeader.PageSize;
var bytesBlock1 = new byte[PageSize];
Expand All @@ -160,7 +163,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH
reader.BaseStream.Seek(PageHeader.BlockBlob * PageSize, SeekOrigin.Begin);
reader.Read(bytesBlock1, 0, PageSize);
DataPage = DatabaseAccess8214.ReadPage(reader, bytesBlock1);
TotalBlocks = DataPage.StorageTables.Sum(ST => ST.DataBlocks.Count);
TotalBlocks = DataPage.StorageTables.Sum(ST => (long)ST.DataBlocks.Count);
bytesBlock = new byte[PageSize * TotalBlocks];
i = 0;
foreach (var ST in DataPage.StorageTables)
Expand All @@ -177,7 +180,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH
}

int NextBlock = DataPos;
int Pos = DataPos * 256;
int Pos = (int)DataPos * 256;
int ii = 0;
while (NextBlock > 0)
{
Expand All @@ -193,6 +196,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH
using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write))
{
using var writer = new BinaryWriter(fs);
long LastPos = 0;
ii = 0;
foreach (var ST in DataPage.StorageTables)
{
Expand All @@ -202,7 +206,9 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH
bytesBlock.AsMemory(ii, TempBlock.Length).CopyTo(TempBlock.AsMemory());
ii += TempBlock.Length;

writer.Seek(DB * PageSize, SeekOrigin.Begin);
long CurPos = DB * PageSize;
int RelativePos = (int)(CurPos - LastPos);
writer.Seek(RelativePos, SeekOrigin.Current);
writer.Write(TempBlock);
}
}
Expand Down
8 changes: 4 additions & 4 deletions PasswordChanger1C/DatabaseAccess8214.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ public static AccessFunctions.PageParams ReadPage(BinaryReader reader, byte[] By
Page.version2 = BitConverter.ToInt32(Bytes, 16);
Page.version = BitConverter.ToInt32(Bytes, 20);
int Index = 24;
Page.PagesNum = new List<int>();
Page.PagesNum = new List<long>();
Page.StorageTables = new List<AccessFunctions.StorageTable>();

// Получим номера страниц размещения
Expand All @@ -104,7 +104,7 @@ public static AccessFunctions.PageParams ReadPage(BinaryReader reader, byte[] By
{
var StorageTables = new AccessFunctions.StorageTable();
StorageTables.Number = blk;
StorageTables.DataBlocks = new List<int>();
StorageTables.DataBlocks = new List<long>();
var bytesBlock = new byte[PageSize];
reader.BaseStream.Seek(blk * PageSize, SeekOrigin.Begin);
reader.Read(bytesBlock, 0, PageSize);
Expand Down Expand Up @@ -132,7 +132,7 @@ public static AccessFunctions.PageParams ReadPage(BinaryReader reader, byte[] By
return Page;
}

public static void ReadDataFromTable(BinaryReader reader, int DB, byte[] bytesBlock, ref AccessFunctions.PageParams PageHeader, string TableNameUsers)
public static void ReadDataFromTable(BinaryReader reader, long DB, byte[] bytesBlock, ref AccessFunctions.PageParams PageHeader, string TableNameUsers)
{
reader.BaseStream.Seek(DB * PageSize, SeekOrigin.Begin);
reader.Read(bytesBlock, 0, PageSize);
Expand Down Expand Up @@ -285,7 +285,7 @@ public static void ReadDataPage(ref AccessFunctions.PageParams PageHeader, strin
int Size = (int)Math.Round(DataPage.Length / (double)PageHeader.RowSize);
for (i = 1; i < Size; i++)
{
int Pos = PageHeader.RowSize * i;
int Pos = (int)PageHeader.RowSize * i;
int FieldStartPos = 0;
bool IsDeleted = BitConverter.ToBoolean(bytesBlock, Pos);
var Dict = new Dictionary<string, object>();
Expand Down
19 changes: 11 additions & 8 deletions PasswordChanger1C/DatabaseAccess838.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ private static AccessFunctions.PageParams FindTableDefinition(BinaryReader reade

private static void ReadAllRecordsFromStoragePages(ref AccessFunctions.PageParams PageHeader, BinaryReader reader)
{
int FirstPage = PageHeader.BlockData;
int BlockBlob = PageHeader.BlockBlob;
long FirstPage = PageHeader.BlockData;
long BlockBlob = PageHeader.BlockBlob;
int PageSize = PageHeader.PageSize;
PageHeader.Records = new List<Dictionary<string, object>>();
var bytesBlock1 = new byte[PageSize];
Expand All @@ -79,10 +79,10 @@ private static void ReadAllRecordsFromStoragePages(ref AccessFunctions.PageParam
var DataPage = ReadObjectPageDefinition(reader, bytesBlock1, PageSize);
DataPage.BinaryData = ReadAllStoragePagesForObject(reader, DataPage);
var bytesBlock = DataPage.BinaryData;
int Size = (int)Math.Round(DataPage.Length / (double)PageHeader.RowSize);
long Size = DataPage.Length / PageHeader.RowSize;
for (int i = 1; i < Size; i++)
{
int Pos = PageHeader.RowSize * i;
int Pos = (int)PageHeader.RowSize * i;
int FieldStartPos = 0;
bool IsDeleted = BitConverter.ToBoolean(bytesBlock, Pos);
var Dict = new Dictionary<string, object>();
Expand Down Expand Up @@ -284,7 +284,7 @@ private static AccessFunctions.PageParams ReadObjectPageDefinition(BinaryReader

Page.Length = BitConverter.ToInt64(Bytes, 16);
int Index = 24;
Page.PagesNum = new List<int>();
Page.PagesNum = new List<long>();

// Получим номера страниц размещения
while (true)
Expand All @@ -309,7 +309,7 @@ private static AccessFunctions.PageParams ReadObjectPageDefinition(BinaryReader
public static void WritePasswordIntoInfoBaseIB(string FileName, AccessFunctions.PageParams PageHeader, byte[] UserID, byte[] OldData, byte[] NewData, int DataPos, int DataSize)
{
int PageSize = PageHeader.PageSize;
int BlockBlob = PageHeader.BlockBlob;
int BlockBlob = (int)PageHeader.BlockBlob;
var BytesBlobBlock = new byte[PageSize];
int[] DataPositions = null;
byte[] BytesValTemp;
Expand Down Expand Up @@ -342,14 +342,17 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, AccessFunctions.
// Blob page(s) has been modified. Let's write it back to database
using var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write);
using var writer = new BinaryWriter(fs);
long LastPos = 0;
CurrentByte = 0;
foreach (var Position in BlobPage.PagesNum)
{
var TempBlock = new byte[PageSize];
BlobPage.BinaryData.AsMemory(CurrentByte, PageSize).CopyTo(TempBlock.AsMemory());
CurrentByte += PageSize;

writer.Seek(Position * PageSize, SeekOrigin.Begin);

long CurPos = Position * PageSize;
int RelativePos = (int)(CurPos - LastPos);
writer.Seek(RelativePos, SeekOrigin.Current);
writer.Write(TempBlock);
}
}
Expand Down

1 comment on commit c13311e

@b4tman
Copy link
Owner Author

@b4tman b4tman commented on c13311e Aug 25, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably fix alex-bochkov#5, alex-bochkov#18

Please sign in to comment.