Force LINQ V2 provider for mongodb provider

LINQ V3 generates a complete block of writing chunk in Mongodb server less or equal to 4.2 due to bug in V3 implementation of LINQ provider. We downgraded provider to V2, also we added publishing symbols for debugging in NUGEt and not only in internal feed in Azure DevOps.
nstoredev · Jul 11, 2023 · 8a1be59 · 8a1be59
1 parent 3bcabd9
commit 8a1be59
Show file tree

Hide file tree

Showing 23 changed files with 286 additions and 94 deletions.
diff --git a/releasenotes.md b/releasenotes.md
@@ -1,5 +1,14 @@
 # vNext
 
+## 0.15.1
+
+- Forced Mongodb drivers to use LINQ 2 provider due to excessive number of bugs for LINQ 3
+- Added support for symbol server outside of azure devops with internal builds.
+
+## 0.14-0.15
+
+- Mostly driver updates.
+
 ## 0.13.0
 
 - Added multi partition read.

diff --git a/src/NStore.BaseSqlPersistence/NStore.BaseSqlPersistence.csproj b/src/NStore.BaseSqlPersistence/NStore.BaseSqlPersistence.csproj
@@ -5,7 +5,13 @@
     <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'true'">netstandard2.0;net48</TargetFrameworks>
     <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
   </PropertyGroup>
@@ -14,6 +20,13 @@
     <None PackagePath="" Include="..\..\LICENSE.md" pack="true" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
   </ItemGroup>

diff --git a/src/NStore.Core.Tests/NStore.Core.Tests.csproj b/src/NStore.Core.Tests/NStore.Core.Tests.csproj
@@ -18,6 +18,10 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Core/NStore.Core.csproj b/src/NStore.Core/NStore.Core.csproj
@@ -6,6 +6,12 @@
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
     <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <AssemblyName>NStore.Core</AssemblyName>
@@ -21,4 +27,10 @@
 
     <None PackagePath="" Include="..\..\LICENSE.md" pack="true" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
 </Project>
diff --git a/src/NStore.Domain.Tests/NStore.Domain.Tests.csproj b/src/NStore.Domain.Tests/NStore.Domain.Tests.csproj
@@ -10,6 +10,10 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Domain/NStore.Domain.csproj b/src/NStore.Domain/NStore.Domain.csproj
@@ -5,7 +5,13 @@
     <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <DebugType>full</DebugType>
@@ -17,6 +23,12 @@
     </Compile>
     <None PackagePath="" Include="..\..\LICENSE.md" pack="true" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
   </ItemGroup>

diff --git a/src/NStore.Persistence.LiteDB.Tests/NStore.Persistence.LiteDB.Tests.csproj b/src/NStore.Persistence.LiteDB.Tests/NStore.Persistence.LiteDB.Tests.csproj
@@ -26,6 +26,10 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Persistence.LiteDB/NStore.Persistence.LiteDB.csproj b/src/NStore.Persistence.LiteDB/NStore.Persistence.LiteDB.csproj
@@ -1,26 +1,33 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'false'">netstandard2.0</TargetFrameworks>
-    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'true'">netstandard2.0;net48</TargetFrameworks>
-    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
-
-    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-    <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
-    <RepositoryType>Git</RepositoryType>
-    <DebugType>full</DebugType>
-    <DebugSymbols>true</DebugSymbols>
-  </PropertyGroup>
-  <ItemGroup>
-    <Compile Include="..\AssemblyInfo.cs">
-      <Link>AssemblyInfo.cs</Link>
-    </Compile>
-  </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="LiteDB" Version="5.0.16" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
-
-  </ItemGroup>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'false'">netstandard2.0</TargetFrameworks>
+    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'true'">netstandard2.0;net48</TargetFrameworks>
+    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
+
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+    <RepositoryType>Git</RepositoryType>
+    <DebugType>full</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="..\AssemblyInfo.cs">
+      <Link>AssemblyInfo.cs</Link>
+    </Compile>
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="LiteDB" Version="5.0.16" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
+
+  </ItemGroup>
 </Project>
diff --git a/src/NStore.Persistence.Mongo.Tests/MongoPersistenceSpecificTests.cs b/src/NStore.Persistence.Mongo.Tests/MongoPersistenceSpecificTests.cs
@@ -245,6 +245,50 @@ public async Task Verify_that_after_append_async_we_have_intercepted_the_call()
         }
     }
 
+    public abstract class insert_id_already_existing_base : BasePersistenceTest
+    {
+        protected internal override MongoPersistenceOptions GetMongoPersistenceOptions()
+        {
+            var options = base.GetMongoPersistenceOptions();
+            options.UseLocalSequence = GetUseLocalSequence();
+            options.SequenceCollectionName = "sequence_test";
+            return options;
+        }
+
+        protected abstract bool GetUseLocalSequence();
+
+        [Fact]
+        public async Task resilient_to_multiple_persistence_write_concurrently()
+        {
+            IPersistence store2 = Create(false);
+            string partition1 = Guid.NewGuid().ToString();
+            string partition2 = Guid.NewGuid().ToString();
+
+            await Store.AppendAsync(partition1, 1, new { data = "first attempt" }).ConfigureAwait(false);
+            //now store2 inserts a chunk with another id 
+            await store2.AppendAsync(partition2, 1, new { data = "first attempt" }).ConfigureAwait(false);
+
+            var chunk = await Store.AppendAsync(partition1, 2, new { data = "second data" }).ConfigureAwait(false);
+            Assert.Equal(3, chunk.Position);
+        }
+    }
+
+    public class insert_id_already_existing_base_local_sequence : insert_id_already_existing_base
+    {
+        protected override bool GetUseLocalSequence()
+        {
+            return true;
+        }
+    }
+
+    public class insert_id_already_existing_base_db_sequence : insert_id_already_existing_base
+    {
+        protected override bool GetUseLocalSequence()
+        {
+            return false;
+        }
+    }
+
     /// <summary>
     /// Correctly initialize the seed when you want to use the sequence generated it
     /// </summary>

diff --git a/src/NStore.Persistence.Mongo.Tests/NStore.Persistence.Mongo.Tests.csproj b/src/NStore.Persistence.Mongo.Tests/NStore.Persistence.Mongo.Tests.csproj
@@ -28,6 +28,10 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Persistence.Mongo/MongoPersistence.cs b/src/NStore.Persistence.Mongo/MongoPersistence.cs
@@ -3,7 +3,6 @@
 using NStore.Core.Logging;
 using NStore.Core.Persistence;
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -72,6 +71,13 @@ private void Connect()
             _options.CustomizePartitionSettings(partitionsBuild);
 
             var settings = MongoClientSettings.FromUrl(partitionsBuild.ToMongoUrl());
+
+            //by default we will use LINQv2. There are bugs in older version of mongo < 4.4
+            //and we feel safer to use the old LINQ provider.
+            settings.LinqProvider = MongoDB.Driver.Linq.LinqProvider.V2;
+
+            //The caller has the option to force the use of other link provider due to customization
+            //of the partition client settings. 
             _options.CustomizePartitionClientSettings(settings);
 
             var partitionsClient = new MongoClient(settings);
@@ -472,6 +478,7 @@ await PushToSubscriber(0, subscription, options, filter, true, cancellationToken
             CancellationToken cancellationToken = default
         )
         {
+            int retry = 0;
             while (true)
             {
                 try
@@ -482,6 +489,13 @@ await _chunks.InsertOneAsync(chunk, cancellationToken: cancellationToken)
                 }
                 catch (MongoWriteException ex)
                 {
+                    //Circuit breaker, if for same reason we cannot write the chunk, we need to stop the process not retrying infiinte times.
+                    if (retry++ > 100)
+                    {
+                        _logger.LogError($"Error During InternalPersistAsync. Reached number of max {retry} retry count: {ex.Message}.\n{ex}");
+                        throw;
+                    }
+
                     //Need to understand what kind of exception we had, some of them could lead to a retry
                     if (ex.WriteError.Category == ServerErrorCategory.DuplicateKey)
                     {
@@ -526,7 +540,7 @@ await _chunks.InsertOneAsync(chunk, cancellationToken: cancellationToken)
                         }
                     }
 
-                    _logger.LogError($"Error During InternalPersistAsync: {ex.Message}.\n{ex.ToString()}");
+                    _logger.LogError($"Error During InternalPersistAsync: {ex.Message}.\n{ex}");
                     throw;
                 }
             }
@@ -598,15 +612,17 @@ private async Task CreateIndexAsync(CancellationToken cancellationToken)
         private async Task ReloadSequence(CancellationToken cancellationToken = default)
         {
             var filter = Builders<TChunk>.Filter.Empty;
-            var lastSequenceNumber = await _chunks
+            var lastRecord = await _chunks
                 .Find(filter)
                 .SortByDescending(x => x.Position)
-                .Project(x => x.Position)
+                .Project(Builders<TChunk>.Projection.Include("_id"))
                 .Limit(1)
-                .FirstOrDefaultAsync(cancellationToken: cancellationToken)
+                .ToCursorAsync()
                 .ConfigureAwait(false);
 
-            this._sequence = lastSequenceNumber;
+            var record = await lastRecord.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+            this._sequence = record == null ? 0 : record["_id"].AsInt64;
         }
 
         private async Task EnsureFirstSequenceRecord()

diff --git a/src/NStore.Persistence.Mongo/NStore.Persistence.Mongo.csproj b/src/NStore.Persistence.Mongo/NStore.Persistence.Mongo.csproj
@@ -6,13 +6,23 @@
 
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <DebugType>full</DebugType>
     <DebugSymbols>true</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="MongoDB.Driver" Version="2.19.2" />
   </ItemGroup>
   <ItemGroup>

diff --git a/src/NStore.Persistence.MsSql.Tests/NStore.Persistence.MsSql.Tests.csproj b/src/NStore.Persistence.MsSql.Tests/NStore.Persistence.MsSql.Tests.csproj
@@ -27,6 +27,10 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0.0" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Persistence.MsSql/NStore.Persistence.MsSql.csproj b/src/NStore.Persistence.MsSql/NStore.Persistence.MsSql.csproj
@@ -6,13 +6,23 @@
 
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <DebugType>full</DebugType>
     <DebugSymbols>true</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
   </ItemGroup>
   <ItemGroup>

diff --git a/src/NStore.Persistence.Sqlite.Tests/NStore.Persistence.Sqlite.Tests.csproj b/src/NStore.Persistence.Sqlite.Tests/NStore.Persistence.Sqlite.Tests.csproj
@@ -30,6 +30,10 @@
         <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.8" />
         <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+        <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
         <PackageReference Include="NETStandard.Library" Version="2.0.3" />
         <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.4" />
         <PackageReference Include="xunit" Version="2.4.2" />