Permalink
Browse files

more ranges => segments

Splitting test projects to unit, integration and ClusterTests
Finished integration on online backup strategies even from the master side.


git-svn-id: https://rhino-tools.svn.sourceforge.net/svnroot/rhino-tools/experiments/dht/dht@2184 079b0acf-d9fa-0310-9935-e5ade295c882
  • Loading branch information...
1 parent cdd5a19 commit a7269f66c3a3484569ff00fe2e2f40c305232d00 ayenderahien committed Jun 5, 2009
Showing with 328 additions and 147 deletions.
  1. 0 ....DistributedHashTable.IntegrationTests → Rhino.DistributedHashTable.ClusterTests}/ClusterTests.cs
  2. 0 ...butedHashTable.IntegrationTests → Rhino.DistributedHashTable.ClusterTests}/FullIntegrationTest.cs
  3. 0 ...utedHashTable.IntegrationTests → Rhino.DistributedHashTable.ClusterTests}/MasterOverTheNetwork.cs
  4. 0 ...ibutedHashTable.IntegrationTests → Rhino.DistributedHashTable.ClusterTests}/NodeOverTheNetwork.cs
  5. +36 −0 Rhino.DistributedHashTable.ClusterTests/Properties/AssemblyInfo.cs
  6. +80 −0 Rhino.DistributedHashTable.ClusterTests/Rhino.DistributedHashTable.ClusterTests.csproj
  7. 0 Rhino.DistributedHashTable.IntegrationTests/{Mini → }/DistributedHashTableReplicationTest.cs
  8. 0 Rhino.DistributedHashTable.IntegrationTests/{Mini → }/DistributedHashTableStorageTest.cs
  9. 0 Rhino.DistributedHashTable.IntegrationTests/{Mini → }/EsentTestBase.cs
  10. 0 Rhino.DistributedHashTable.IntegrationTests/{Mini → }/OnlineRangeReplicationCommandTest.cs
  11. +4 −11 Rhino.DistributedHashTable.IntegrationTests/Rhino.DistributedHashTable.IntegrationTests.csproj
  12. +0 −15 Rhino.DistributedHashTable.IntegrationTests/app.config
  13. +21 −0 Rhino.DistributedHashTable.Tests/MasterCaughtUpBehavior.cs
  14. +17 −0 Rhino.DistributedHashTable.Tests/MasterGaveUpBehavior.cs
  15. +6 −0 Rhino.DistributedHashTable.sln
  16. +2 −2 Rhino.DistributedHashTable/Client/DistributedHashTableMasterClient.cs
  17. +4 −4 Rhino.DistributedHashTable/Client/DistributedHashTableStorageClient.cs
  18. +16 −16 Rhino.DistributedHashTable/Commands/OnlineRangeReplicationCommand.cs
  19. +4 −4 Rhino.DistributedHashTable/Commands/RearrangeBackups.cs
  20. +89 −46 Rhino.DistributedHashTable/Internal/DistributedHashTableMaster.cs
  21. +10 −10 Rhino.DistributedHashTable/Internal/DistributedHashTableNode.cs
  22. +5 −5 Rhino.DistributedHashTable/Internal/IDistributedHashTableMaster.cs
  23. +5 −5 Rhino.DistributedHashTable/Internal/IDistributedHashTableNode.cs
  24. +8 −8 Rhino.DistributedHashTable/Internal/Topology.cs
  25. +10 −10 Rhino.DistributedHashTable/Remote/DistributedHashTableNodeReplication.cs
  26. +8 −8 Rhino.DistributedHashTable/Remote/IDistributedHashTableNodeReplication.cs
  27. +3 −3 Rhino.DistributedHashTable/Todo.txt
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Rhino.DistributedHashTable.ClusterTests")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("Rhino.DistributedHashTable.ClusterTests")]
+[assembly: AssemblyCopyright("Copyright © Microsoft 2009")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("0cedc767-aa73-481b-ac16-ad6d5d4cace9")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="3.5" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+ <ProductVersion>9.0.30729</ProductVersion>
+ <SchemaVersion>2.0</SchemaVersion>
+ <ProjectGuid>{D9377684-AC14-47F6-A3E8-E84AD0FC2BAA}</ProjectGuid>
+ <OutputType>Library</OutputType>
+ <AppDesignerFolder>Properties</AppDesignerFolder>
+ <RootNamespace>Rhino.DistributedHashTable.ClusterTests</RootNamespace>
+ <AssemblyName>Rhino.DistributedHashTable.ClusterTests</AssemblyName>
+ <TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
+ <FileAlignment>512</FileAlignment>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+ <DebugSymbols>true</DebugSymbols>
+ <DebugType>full</DebugType>
+ <Optimize>false</Optimize>
+ <OutputPath>bin\Debug\</OutputPath>
+ <DefineConstants>DEBUG;TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
+ <OutputPath>bin\Release\</OutputPath>
+ <DefineConstants>TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <ItemGroup>
+ <Reference Include="log4net, Version=1.2.10.0, Culture=neutral, PublicKeyToken=1b44e1d426115821, processorArchitecture=MSIL">
+ <SpecificVersion>False</SpecificVersion>
+ <HintPath>..\Rhino.DistributedHashTable\bin\Debug\log4net.dll</HintPath>
+ </Reference>
+ <Reference Include="System" />
+ <Reference Include="System.Core">
+ <RequiredTargetFramework>3.5</RequiredTargetFramework>
+ </Reference>
+ <Reference Include="System.Xml.Linq">
+ <RequiredTargetFramework>3.5</RequiredTargetFramework>
+ </Reference>
+ <Reference Include="System.Data.DataSetExtensions">
+ <RequiredTargetFramework>3.5</RequiredTargetFramework>
+ </Reference>
+ <Reference Include="System.Data" />
+ <Reference Include="System.Xml" />
+ <Reference Include="xunit, Version=1.1.0.1323, Culture=neutral, PublicKeyToken=8d05b1bb7a6fdb6c, processorArchitecture=MSIL">
+ <SpecificVersion>False</SpecificVersion>
+ <HintPath>..\Rhino.DistributedHashTable.Tests\bin\Debug\xunit.dll</HintPath>
+ </Reference>
+ </ItemGroup>
+ <ItemGroup>
+ <Compile Include="ClusterTests.cs" />
+ <Compile Include="FullIntegrationTest.cs" />
+ <Compile Include="MasterOverTheNetwork.cs" />
+ <Compile Include="NodeOverTheNetwork.cs" />
+ <Compile Include="Properties\AssemblyInfo.cs" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\..\pht\Rhino.PersistentHashTable\Rhino.PersistentHashTable.csproj">
+ <Project>{F30B2D63-CED5-4C8A-908F-0B5503D984A9}</Project>
+ <Name>Rhino.PersistentHashTable</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Rhino.DistributedHashTable\Rhino.DistributedHashTable.csproj">
+ <Project>{4E8D44D2-505D-488C-B92C-51147748B104}</Project>
+ <Name>Rhino.DistributedHashTable</Name>
+ </ProjectReference>
+ </ItemGroup>
+ <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+ <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Microsoft.Common.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
@@ -60,15 +60,11 @@
</Reference>
</ItemGroup>
<ItemGroup>
- <Compile Include="ClusterTests.cs" />
- <Compile Include="FullIntegrationTest.cs" />
- <Compile Include="MasterOverTheNetwork.cs" />
- <Compile Include="Mini\DistributedHashTableReplicationTest.cs" />
- <Compile Include="Mini\DistributedHashTableStorageTest.cs" />
- <Compile Include="Mini\EsentTestBase.cs" />
- <Compile Include="Mini\OnlineRangeReplicationCommandTest.cs" />
+ <Compile Include="DistributedHashTableReplicationTest.cs" />
+ <Compile Include="DistributedHashTableStorageTest.cs" />
+ <Compile Include="EsentTestBase.cs" />
+ <Compile Include="OnlineRangeReplicationCommandTest.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
- <Compile Include="NodeOverTheNetwork.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\pht\Rhino.PersistentHashTable\Rhino.PersistentHashTable.csproj">
@@ -80,9 +76,6 @@
<Name>Rhino.DistributedHashTable</Name>
</ProjectReference>
</ItemGroup>
- <ItemGroup>
- <None Include="app.config" />
- </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
@@ -1,15 +0,0 @@
-<configuration>
- <system.diagnostics>
- <sources>
- <source name="System.ServiceModel"
- switchValue="Information, ActivityTracing"
- propagateActivity="true">
- <listeners>
- <add name="traceListener"
- type="System.Diagnostics.XmlWriterTraceListener"
- initializeData= "Traces.svclog" />
- </listeners>
- </source>
- </sources>
- </system.diagnostics>
-</configuration>
@@ -28,6 +28,27 @@ public void WillRaiseTopologyChangedEvent()
master.CaughtUp(endPoint, ReplicationType.Ownership, ranges.First().Index);
Assert.True(wasCalled);
}
+
+ [Fact]
+ public void WhenCatchingUpOnBackupsWillMoveFromPendingBackupsToBackups()
+ {
+ master.CaughtUp(endPoint, ReplicationType.Ownership,
+ master.Join(endPoint).Select(x => x.Index).ToArray());
+
+ var anotherEndpoint = NodeEndpoint.ForTest(54);
+ master.CaughtUp(anotherEndpoint, ReplicationType.Ownership,
+ master.Join(anotherEndpoint).Select(x => x.Index).ToArray());
+
+ var segment = master.Topology.Segments.First(x => x.PendingBackups.Count > 0);
+
+ master.CaughtUp(segment.PendingBackups.First(), ReplicationType.Backup,
+ segment.Index);
+
+ segment = master.Topology.Segments[segment.Index];
+
+ Assert.Empty(segment.PendingBackups);
+ Assert.Equal(1, segment.Backups.Count);
+ }
}
}
}
@@ -32,6 +32,23 @@ public void WillRemoveThePendingMoveFromTheSegment()
Assert.Null(range.InProcessOfMovingToEndpoint);
}
+
+ [Fact]
+ public void WhenGivingUpOnBackupsWillRemoveFromPendingBackups()
+ {
+ master.CaughtUp(endPoint, ReplicationType.Ownership,
+ master.Join(endPoint).Select(x => x.Index).ToArray());
+
+ var segment = master.Topology.Segments.First(x => x.PendingBackups.Count > 0);
+
+ master.GaveUp(segment.PendingBackups.First(), ReplicationType.Backup,
+ segment.Index);
+
+ segment = master.Topology.Segments[segment.Index];
+
+ Assert.Empty(segment.PendingBackups);
+ Assert.Equal(0, segment.Backups.Count);
+ }
}
}
}
@@ -15,6 +15,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Rhino.Queues", "..\queues\R
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Rhino.Queues.Tests", "..\queues\Rhino.Queues.Tests\Rhino.Queues.Tests.csproj", "{98D394CA-9FEF-49AD-8DAB-FE039152E001}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Rhino.DistributedHashTable.ClusterTests", "Rhino.DistributedHashTable.ClusterTests\Rhino.DistributedHashTable.ClusterTests.csproj", "{D9377684-AC14-47F6-A3E8-E84AD0FC2BAA}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -49,6 +51,10 @@ Global
{98D394CA-9FEF-49AD-8DAB-FE039152E001}.Debug|Any CPU.Build.0 = Debug|Any CPU
{98D394CA-9FEF-49AD-8DAB-FE039152E001}.Release|Any CPU.ActiveCfg = Release|Any CPU
{98D394CA-9FEF-49AD-8DAB-FE039152E001}.Release|Any CPU.Build.0 = Release|Any CPU
+ {D9377684-AC14-47F6-A3E8-E84AD0FC2BAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {D9377684-AC14-47F6-A3E8-E84AD0FC2BAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {D9377684-AC14-47F6-A3E8-E84AD0FC2BAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {D9377684-AC14-47F6-A3E8-E84AD0FC2BAA}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -130,7 +130,7 @@ public Topology GetTopology()
public void GaveUp(NodeEndpoint endpoint,
ReplicationType type,
- params int[] rangesGivingUpOn)
+ params int[] segmentsGivingUpOn)
{
Execute((writer,
stream) =>
@@ -140,7 +140,7 @@ public Topology GetTopology()
Type = MasterMessageType.GaveUpRequest,
GaveUp = new GaveUpRequestMessage.Builder
{
- GaveUpSegmentsList = { rangesGivingUpOn },
+ GaveUpSegmentsList = { segmentsGivingUpOn },
Type = type == ReplicationType.Backup ? Protocol.ReplicationType.Backup : Protocol.ReplicationType.Ownership,
Endpoint = new Protocol.NodeEndpoint.Builder
{
@@ -171,7 +171,7 @@ public IDistributedHashTableNodeReplication Replication
}
public ReplicationResult ReplicateNextPage(NodeEndpoint replicationEndpoint,
- int range)
+ int segment)
{
writer.Write(new StorageMessageUnion.Builder
{
@@ -183,7 +183,7 @@ public IDistributedHashTableNodeReplication Replication
Async = replicationEndpoint.Async.ToString(),
Sync = replicationEndpoint.Sync.ToString()
}.Build(),
- Segment = range
+ Segment = segment
}.Build()
}.Build());
writer.Flush();
@@ -204,7 +204,7 @@ public IDistributedHashTableNodeReplication Replication
}
public int[] AssignAllEmptySegments(NodeEndpoint replicationEndpoint,
- int[] ranges)
+ int[] segments)
{
writer.Write(new StorageMessageUnion.Builder
{
@@ -216,7 +216,7 @@ public IDistributedHashTableNodeReplication Replication
Async = replicationEndpoint.Async.ToString(),
Sync = replicationEndpoint.Sync.ToString()
}.Build(),
- SegmentsList = { ranges }
+ SegmentsList = { segments }
}.Build()
}.Build());
writer.Flush();
@@ -49,10 +49,10 @@ public bool Execute()
return false;
try
{
- var rangesToLoad = AssignAllEmptySegmentsFromEndpoint(processedSegments);
+ var segmentsToLoad = AssignAllEmptySegmentsFromEndpoint(processedSegments);
if (continueWorking == false)
return false;
- return ProcessSegmentsWithData(rangesToLoad, processedSegments) == false;
+ return ProcessSegmentsWithData(segmentsToLoad, processedSegments) == false;
}
catch (Exception e)
{
@@ -85,49 +85,49 @@ public bool Execute()
}
}
- private bool ProcessSegmentsWithData(IEnumerable<Segment> rangesToLoad,
+ private bool ProcessSegmentsWithData(IEnumerable<Segment> segmentsToLoad,
ICollection<int> processedSegments)
{
var someFailed = false;
int numberOfFailures = 0;
- foreach (var range in rangesToLoad)
+ foreach (var segment in segmentsToLoad)
{
if (continueWorking == false)
return true;
try
{
- ReplicateSegment(range);
- processedSegments.Add(range.Index);
+ ReplicateSegment(segment);
+ processedSegments.Add(segment.Index);
}
catch (Exception e)
{
- log.Error("Failed to replicate range " + range, e);
+ log.Error("Failed to replicate segment " + segment, e);
numberOfFailures += 1;
if (numberOfFailures > 5)
{
log.WarnFormat("Failed to replicate {0} times, giving up on all additional segments",
numberOfFailures);
break;
}
- node.GivingUpOn(type, range.Index);
- processedSegments.Add(range.Index);
+ node.GivingUpOn(type, segment.Index);
+ processedSegments.Add(segment.Index);
someFailed |= true;
}
}
return someFailed;
}
- private void ReplicateSegment(Segment range)
+ private void ReplicateSegment(Segment segment)
{
while (true)
{
- log.DebugFormat("Starting replication of range [{0}] from {1}",
- range,
+ log.DebugFormat("Starting replication of segment [{0}] from {1}",
+ segment,
endpoint);
- var result = otherNode.ReplicateNextPage(node.Endpoint, range.Index);
- log.DebugFormat("Replication of range [{0}] from {1} got {2} puts & {3} removals",
- range,
+ var result = otherNode.ReplicateNextPage(node.Endpoint, segment.Index);
+ log.DebugFormat("Replication of segment [{0}] from {1} got {2} puts & {3} removals",
+ segment,
endpoint,
result.PutRequests.Length,
result.RemoveRequests.Length);
@@ -141,7 +141,7 @@ private void ReplicateSegment(Segment range)
if (result.Done)
break;
}
- node.DoneReplicatingSegments(type, range.Index);
+ node.DoneReplicatingSegments(type, segment.Index);
}
private List<Segment> AssignAllEmptySegmentsFromEndpoint(List<int> processedSegments)
Oops, something went wrong.

0 comments on commit a7269f6

Please sign in to comment.