Merge tag '0.15.1' into develop

Fix for mongodb
nstoredev · Jul 11, 2023 · 893fbb7 · 893fbb7
2 parents 3bcabd9 + 56577c0
commit 893fbb7
Show file tree

Hide file tree

Showing 23 changed files with 286 additions and 94 deletions.
diff --git a/releasenotes.md b/releasenotes.md
@@ -1,5 +1,14 @@
 # vNext
 
+## 0.15.1
+
+- Forced Mongodb drivers to use LINQ 2 provider due to excessive number of bugs for LINQ 3
+- Added support for symbol server outside of azure devops with internal builds.
+
+## 0.14-0.15
+
+- Mostly driver updates.
+
 ## 0.13.0
 
 - Added multi partition read.

diff --git a/src/NStore.BaseSqlPersistence/NStore.BaseSqlPersistence.csproj b/src/NStore.BaseSqlPersistence/NStore.BaseSqlPersistence.csproj
@@ -5,7 +5,13 @@
     <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'true'">netstandard2.0;net48</TargetFrameworks>
     <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
   </PropertyGroup>
@@ -14,6 +20,13 @@
     <None PackagePath="" Include="..\..\LICENSE.md" pack="true" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
   </ItemGroup>

diff --git a/src/NStore.Core.Tests/NStore.Core.Tests.csproj b/src/NStore.Core.Tests/NStore.Core.Tests.csproj
@@ -18,6 +18,10 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Core/NStore.Core.csproj b/src/NStore.Core/NStore.Core.csproj
@@ -6,6 +6,12 @@
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
     <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <AssemblyName>NStore.Core</AssemblyName>
@@ -21,4 +27,10 @@
 
     <None PackagePath="" Include="..\..\LICENSE.md" pack="true" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
 </Project>
diff --git a/src/NStore.Domain.Tests/NStore.Domain.Tests.csproj b/src/NStore.Domain.Tests/NStore.Domain.Tests.csproj
@@ -10,6 +10,10 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Domain/NStore.Domain.csproj b/src/NStore.Domain/NStore.Domain.csproj
@@ -5,7 +5,13 @@
     <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <DebugType>full</DebugType>
@@ -17,6 +23,12 @@
     </Compile>
     <None PackagePath="" Include="..\..\LICENSE.md" pack="true" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
   </ItemGroup>

diff --git a/src/NStore.Persistence.LiteDB.Tests/NStore.Persistence.LiteDB.Tests.csproj b/src/NStore.Persistence.LiteDB.Tests/NStore.Persistence.LiteDB.Tests.csproj
@@ -26,6 +26,10 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Persistence.LiteDB/NStore.Persistence.LiteDB.csproj b/src/NStore.Persistence.LiteDB/NStore.Persistence.LiteDB.csproj
@@ -1,26 +1,33 @@
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'false'">netstandard2.0</TargetFrameworks>
-    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'true'">netstandard2.0;net48</TargetFrameworks>
-    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
-
-    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
-    <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
-    <RepositoryType>Git</RepositoryType>
-    <DebugType>full</DebugType>
-    <DebugSymbols>true</DebugSymbols>
-  </PropertyGroup>
-  <ItemGroup>
-    <Compile Include="..\AssemblyInfo.cs">
-      <Link>AssemblyInfo.cs</Link>
-    </Compile>
-  </ItemGroup>
-  <ItemGroup>
-    <PackageReference Include="LiteDB" Version="5.0.16" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
-
-  </ItemGroup>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'false'">netstandard2.0</TargetFrameworks>
+    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == 'true'">netstandard2.0;net48</TargetFrameworks>
+    <TargetFrameworks Condition="'$(DOTNETCORE_MULTITARGET)' == ''">netstandard2.0</TargetFrameworks>
+
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+    <RepositoryType>Git</RepositoryType>
+    <DebugType>full</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="..\AssemblyInfo.cs">
+      <Link>AssemblyInfo.cs</Link>
+    </Compile>
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="LiteDB" Version="5.0.16" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\NStore.Core\NStore.Core.csproj" />
+
+  </ItemGroup>
 </Project>
diff --git a/src/NStore.Persistence.Mongo.Tests/MongoPersistenceSpecificTests.cs b/src/NStore.Persistence.Mongo.Tests/MongoPersistenceSpecificTests.cs
@@ -245,6 +245,50 @@ public async Task Verify_that_after_append_async_we_have_intercepted_the_call()
         }
     }
 
+    public abstract class insert_id_already_existing_base : BasePersistenceTest
+    {
+        protected internal override MongoPersistenceOptions GetMongoPersistenceOptions()
+        {
+            var options = base.GetMongoPersistenceOptions();
+            options.UseLocalSequence = GetUseLocalSequence();
+            options.SequenceCollectionName = "sequence_test";
+            return options;
+        }
+
+        protected abstract bool GetUseLocalSequence();
+
+        [Fact]
+        public async Task resilient_to_multiple_persistence_write_concurrently()
+        {
+            IPersistence store2 = Create(false);
+            string partition1 = Guid.NewGuid().ToString();
+            string partition2 = Guid.NewGuid().ToString();
+
+            await Store.AppendAsync(partition1, 1, new { data = "first attempt" }).ConfigureAwait(false);
+            //now store2 inserts a chunk with another id 
+            await store2.AppendAsync(partition2, 1, new { data = "first attempt" }).ConfigureAwait(false);
+
+            var chunk = await Store.AppendAsync(partition1, 2, new { data = "second data" }).ConfigureAwait(false);
+            Assert.Equal(3, chunk.Position);
+        }
+    }
+
+    public class insert_id_already_existing_base_local_sequence : insert_id_already_existing_base
+    {
+        protected override bool GetUseLocalSequence()
+        {
+            return true;
+        }
+    }
+
+    public class insert_id_already_existing_base_db_sequence : insert_id_already_existing_base
+    {
+        protected override bool GetUseLocalSequence()
+        {
+            return false;
+        }
+    }
+
     /// <summary>
     /// Correctly initialize the seed when you want to use the sequence generated it
     /// </summary>

diff --git a/src/NStore.Persistence.Mongo.Tests/NStore.Persistence.Mongo.Tests.csproj b/src/NStore.Persistence.Mongo.Tests/NStore.Persistence.Mongo.Tests.csproj
@@ -28,6 +28,10 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Persistence.Mongo/MongoPersistence.cs b/src/NStore.Persistence.Mongo/MongoPersistence.cs
@@ -3,7 +3,6 @@
 using NStore.Core.Logging;
 using NStore.Core.Persistence;
 using System;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -72,6 +71,13 @@ private void Connect()
             _options.CustomizePartitionSettings(partitionsBuild);
 
             var settings = MongoClientSettings.FromUrl(partitionsBuild.ToMongoUrl());
+
+            //by default we will use LINQv2. There are bugs in older version of mongo < 4.4
+            //and we feel safer to use the old LINQ provider.
+            settings.LinqProvider = MongoDB.Driver.Linq.LinqProvider.V2;
+
+            //The caller has the option to force the use of other link provider due to customization
+            //of the partition client settings. 
             _options.CustomizePartitionClientSettings(settings);
 
             var partitionsClient = new MongoClient(settings);
@@ -472,6 +478,7 @@ await PushToSubscriber(0, subscription, options, filter, true, cancellationToken
             CancellationToken cancellationToken = default
         )
         {
+            int retry = 0;
             while (true)
             {
                 try
@@ -482,6 +489,13 @@ await _chunks.InsertOneAsync(chunk, cancellationToken: cancellationToken)
                 }
                 catch (MongoWriteException ex)
                 {
+                    //Circuit breaker, if for same reason we cannot write the chunk, we need to stop the process not retrying infiinte times.
+                    if (retry++ > 100)
+                    {
+                        _logger.LogError($"Error During InternalPersistAsync. Reached number of max {retry} retry count: {ex.Message}.\n{ex}");
+                        throw;
+                    }
+
                     //Need to understand what kind of exception we had, some of them could lead to a retry
                     if (ex.WriteError.Category == ServerErrorCategory.DuplicateKey)
                     {
@@ -526,7 +540,7 @@ await _chunks.InsertOneAsync(chunk, cancellationToken: cancellationToken)
                         }
                     }
 
-                    _logger.LogError($"Error During InternalPersistAsync: {ex.Message}.\n{ex.ToString()}");
+                    _logger.LogError($"Error During InternalPersistAsync: {ex.Message}.\n{ex}");
                     throw;
                 }
             }
@@ -598,15 +612,17 @@ private async Task CreateIndexAsync(CancellationToken cancellationToken)
         private async Task ReloadSequence(CancellationToken cancellationToken = default)
         {
             var filter = Builders<TChunk>.Filter.Empty;
-            var lastSequenceNumber = await _chunks
+            var lastRecord = await _chunks
                 .Find(filter)
                 .SortByDescending(x => x.Position)
-                .Project(x => x.Position)
+                .Project(Builders<TChunk>.Projection.Include("_id"))
                 .Limit(1)
-                .FirstOrDefaultAsync(cancellationToken: cancellationToken)
+                .ToCursorAsync()
                 .ConfigureAwait(false);
 
-            this._sequence = lastSequenceNumber;
+            var record = await lastRecord.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+
+            this._sequence = record == null ? 0 : record["_id"].AsInt64;
         }
 
         private async Task EnsureFirstSequenceRecord()

diff --git a/src/NStore.Persistence.Mongo/NStore.Persistence.Mongo.csproj b/src/NStore.Persistence.Mongo/NStore.Persistence.Mongo.csproj
@@ -6,13 +6,23 @@
 
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <DebugType>full</DebugType>
     <DebugSymbols>true</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="MongoDB.Driver" Version="2.19.2" />
   </ItemGroup>
   <ItemGroup>

diff --git a/src/NStore.Persistence.MsSql.Tests/NStore.Persistence.MsSql.Tests.csproj b/src/NStore.Persistence.MsSql.Tests/NStore.Persistence.MsSql.Tests.csproj
@@ -27,6 +27,10 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="NETStandard.Library" Version="2.0.3" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0.0" />
     <PackageReference Include="xunit" Version="2.4.2" />

diff --git a/src/NStore.Persistence.MsSql/NStore.Persistence.MsSql.csproj b/src/NStore.Persistence.MsSql/NStore.Persistence.MsSql.csproj
@@ -6,13 +6,23 @@
 
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <PackageProjectUrl>https://github.com/ProximoSrl/NStore</PackageProjectUrl>
-    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+    <RepositoryUrl>https://github.com/ProximoSrl/NStore</RepositoryUrl>
+        <PublishRepositoryUrl>true</PublishRepositoryUrl>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+   <Deterministic>true</Deterministic>
+   <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">True</ContinuousIntegrationBuild>
+   <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">True</ContinuousIntegrationBuild> 
     <RepositoryType>Git</RepositoryType>
     <PackageLicenseFile>LICENSE.md</PackageLicenseFile>
     <DebugType>full</DebugType>
     <DebugSymbols>true</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="System.Data.SqlClient" Version="4.8.5" />
   </ItemGroup>
   <ItemGroup>

diff --git a/src/NStore.Persistence.Sqlite.Tests/NStore.Persistence.Sqlite.Tests.csproj b/src/NStore.Persistence.Sqlite.Tests/NStore.Persistence.Sqlite.Tests.csproj
@@ -30,6 +30,10 @@
         <PackageReference Include="Microsoft.Data.Sqlite" Version="6.0.8" />
         <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="2.0.0" />
         <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+        <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.1.1">
+          <PrivateAssets>all</PrivateAssets>
+          <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
         <PackageReference Include="NETStandard.Library" Version="2.0.3" />
         <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.4" />
         <PackageReference Include="xunit" Version="2.4.2" />