From c13311e6138ff6f91e09bcbcac4ce61d134df535 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Wed, 11 Aug 2021 21:48:33 +0300 Subject: [PATCH] work with large files (~15GB tested) --- PasswordChanger1C/AccessFunctions.cs | 34 +++++++++++++++---------- PasswordChanger1C/DatabaseAccess8214.cs | 8 +++--- PasswordChanger1C/DatabaseAccess838.cs | 19 ++++++++------ 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/PasswordChanger1C/AccessFunctions.cs b/PasswordChanger1C/AccessFunctions.cs index 2f566e5..9d02409 100644 --- a/PasswordChanger1C/AccessFunctions.cs +++ b/PasswordChanger1C/AccessFunctions.cs @@ -10,8 +10,8 @@ public static class AccessFunctions { public struct StorageTable { - public int Number; - public List DataBlocks; + public long Number; + public List DataBlocks; } public struct TableFields @@ -20,7 +20,7 @@ public struct TableFields public int Length; public int Precision; public int Size; - public int Offset; + public long Offset; public string Type; public int CouldBeNull; } @@ -34,13 +34,13 @@ public struct PageParams public int version1; public int version2; public int version; - public List PagesNum; + public List PagesNum; public List StorageTables; public List Fields; - public int RowSize; + public long RowSize; public List> Records; - public int BlockData; - public int BlockBlob; + public long BlockData; + public long BlockBlob; public int PageSize; public string TableDefinition; public string DatabaseVersion; @@ -89,7 +89,7 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag { byte[] bytesBlock; PageParams DataPage; - int TotalBlocks; + long TotalBlocks; int i; using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write)) @@ -100,7 +100,7 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag reader.BaseStream.Seek(PageHeader.BlockData * 4096, SeekOrigin.Begin); reader.Read(bytesBlock1, 0, 4096); DataPage = DatabaseAccess8214.ReadPage(reader, bytesBlock1); - TotalBlocks = DataPage.StorageTables.Sum(ST => ST.DataBlocks.Count); + TotalBlocks = DataPage.StorageTables.Sum(ST => (long)ST.DataBlocks.Count); bytesBlock = new byte[4096 * TotalBlocks]; i = 0; foreach (var ST in DataPage.StorageTables) @@ -123,6 +123,7 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write)) { using var writer = new BinaryWriter(fs); + long LastPos = 0; i = 0; foreach (var ST in DataPage.StorageTables) { @@ -132,7 +133,9 @@ public static void WritePasswordIntoInfoBaseRepo(string FileName, PageParams Pag bytesBlock.AsMemory(i, TempBlock.Length).CopyTo(TempBlock.AsMemory()); i += TempBlock.Length; - writer.Seek(DB * 4096, SeekOrigin.Begin); + long CurPos = DB * 4096; + int RelativePos = (int)(CurPos - LastPos); + writer.Seek(RelativePos, SeekOrigin.Current); writer.Write(TempBlock); } } @@ -148,7 +151,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH } int i; - int TotalBlocks; + long TotalBlocks; byte[] bytesBlock; int PageSize = PageHeader.PageSize; var bytesBlock1 = new byte[PageSize]; @@ -160,7 +163,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH reader.BaseStream.Seek(PageHeader.BlockBlob * PageSize, SeekOrigin.Begin); reader.Read(bytesBlock1, 0, PageSize); DataPage = DatabaseAccess8214.ReadPage(reader, bytesBlock1); - TotalBlocks = DataPage.StorageTables.Sum(ST => ST.DataBlocks.Count); + TotalBlocks = DataPage.StorageTables.Sum(ST => (long)ST.DataBlocks.Count); bytesBlock = new byte[PageSize * TotalBlocks]; i = 0; foreach (var ST in DataPage.StorageTables) @@ -177,7 +180,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH } int NextBlock = DataPos; - int Pos = DataPos * 256; + int Pos = (int)DataPos * 256; int ii = 0; while (NextBlock > 0) { @@ -193,6 +196,7 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH using (var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write)) { using var writer = new BinaryWriter(fs); + long LastPos = 0; ii = 0; foreach (var ST in DataPage.StorageTables) { @@ -202,7 +206,9 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, PageParams PageH bytesBlock.AsMemory(ii, TempBlock.Length).CopyTo(TempBlock.AsMemory()); ii += TempBlock.Length; - writer.Seek(DB * PageSize, SeekOrigin.Begin); + long CurPos = DB * PageSize; + int RelativePos = (int)(CurPos - LastPos); + writer.Seek(RelativePos, SeekOrigin.Current); writer.Write(TempBlock); } } diff --git a/PasswordChanger1C/DatabaseAccess8214.cs b/PasswordChanger1C/DatabaseAccess8214.cs index 09f2f8b..7b29adc 100644 --- a/PasswordChanger1C/DatabaseAccess8214.cs +++ b/PasswordChanger1C/DatabaseAccess8214.cs @@ -80,7 +80,7 @@ public static AccessFunctions.PageParams ReadPage(BinaryReader reader, byte[] By Page.version2 = BitConverter.ToInt32(Bytes, 16); Page.version = BitConverter.ToInt32(Bytes, 20); int Index = 24; - Page.PagesNum = new List(); + Page.PagesNum = new List(); Page.StorageTables = new List(); // Получим номера страниц размещения @@ -104,7 +104,7 @@ public static AccessFunctions.PageParams ReadPage(BinaryReader reader, byte[] By { var StorageTables = new AccessFunctions.StorageTable(); StorageTables.Number = blk; - StorageTables.DataBlocks = new List(); + StorageTables.DataBlocks = new List(); var bytesBlock = new byte[PageSize]; reader.BaseStream.Seek(blk * PageSize, SeekOrigin.Begin); reader.Read(bytesBlock, 0, PageSize); @@ -132,7 +132,7 @@ public static AccessFunctions.PageParams ReadPage(BinaryReader reader, byte[] By return Page; } - public static void ReadDataFromTable(BinaryReader reader, int DB, byte[] bytesBlock, ref AccessFunctions.PageParams PageHeader, string TableNameUsers) + public static void ReadDataFromTable(BinaryReader reader, long DB, byte[] bytesBlock, ref AccessFunctions.PageParams PageHeader, string TableNameUsers) { reader.BaseStream.Seek(DB * PageSize, SeekOrigin.Begin); reader.Read(bytesBlock, 0, PageSize); @@ -285,7 +285,7 @@ public static void ReadDataPage(ref AccessFunctions.PageParams PageHeader, strin int Size = (int)Math.Round(DataPage.Length / (double)PageHeader.RowSize); for (i = 1; i < Size; i++) { - int Pos = PageHeader.RowSize * i; + int Pos = (int)PageHeader.RowSize * i; int FieldStartPos = 0; bool IsDeleted = BitConverter.ToBoolean(bytesBlock, Pos); var Dict = new Dictionary(); diff --git a/PasswordChanger1C/DatabaseAccess838.cs b/PasswordChanger1C/DatabaseAccess838.cs index c34f1ff..11214ab 100644 --- a/PasswordChanger1C/DatabaseAccess838.cs +++ b/PasswordChanger1C/DatabaseAccess838.cs @@ -69,8 +69,8 @@ private static AccessFunctions.PageParams FindTableDefinition(BinaryReader reade private static void ReadAllRecordsFromStoragePages(ref AccessFunctions.PageParams PageHeader, BinaryReader reader) { - int FirstPage = PageHeader.BlockData; - int BlockBlob = PageHeader.BlockBlob; + long FirstPage = PageHeader.BlockData; + long BlockBlob = PageHeader.BlockBlob; int PageSize = PageHeader.PageSize; PageHeader.Records = new List>(); var bytesBlock1 = new byte[PageSize]; @@ -79,10 +79,10 @@ private static void ReadAllRecordsFromStoragePages(ref AccessFunctions.PageParam var DataPage = ReadObjectPageDefinition(reader, bytesBlock1, PageSize); DataPage.BinaryData = ReadAllStoragePagesForObject(reader, DataPage); var bytesBlock = DataPage.BinaryData; - int Size = (int)Math.Round(DataPage.Length / (double)PageHeader.RowSize); + long Size = DataPage.Length / PageHeader.RowSize; for (int i = 1; i < Size; i++) { - int Pos = PageHeader.RowSize * i; + int Pos = (int)PageHeader.RowSize * i; int FieldStartPos = 0; bool IsDeleted = BitConverter.ToBoolean(bytesBlock, Pos); var Dict = new Dictionary(); @@ -284,7 +284,7 @@ private static AccessFunctions.PageParams ReadObjectPageDefinition(BinaryReader Page.Length = BitConverter.ToInt64(Bytes, 16); int Index = 24; - Page.PagesNum = new List(); + Page.PagesNum = new List(); // Получим номера страниц размещения while (true) @@ -309,7 +309,7 @@ private static AccessFunctions.PageParams ReadObjectPageDefinition(BinaryReader public static void WritePasswordIntoInfoBaseIB(string FileName, AccessFunctions.PageParams PageHeader, byte[] UserID, byte[] OldData, byte[] NewData, int DataPos, int DataSize) { int PageSize = PageHeader.PageSize; - int BlockBlob = PageHeader.BlockBlob; + int BlockBlob = (int)PageHeader.BlockBlob; var BytesBlobBlock = new byte[PageSize]; int[] DataPositions = null; byte[] BytesValTemp; @@ -342,14 +342,17 @@ public static void WritePasswordIntoInfoBaseIB(string FileName, AccessFunctions. // Blob page(s) has been modified. Let's write it back to database using var fs = new FileStream(FileName, FileMode.Open, FileAccess.ReadWrite, FileShare.Write); using var writer = new BinaryWriter(fs); + long LastPos = 0; CurrentByte = 0; foreach (var Position in BlobPage.PagesNum) { var TempBlock = new byte[PageSize]; BlobPage.BinaryData.AsMemory(CurrentByte, PageSize).CopyTo(TempBlock.AsMemory()); CurrentByte += PageSize; - - writer.Seek(Position * PageSize, SeekOrigin.Begin); + + long CurPos = Position * PageSize; + int RelativePos = (int)(CurPos - LastPos); + writer.Seek(RelativePos, SeekOrigin.Current); writer.Write(TempBlock); } }