Skip to content

Commit

Permalink
NCBC-1337: When a timeout occurs Muxio returns Success with no body
Browse files Browse the repository at this point in the history
Motivation
----------
If a connection is terminated by the server, any in-flight operations must
be canceled and the response should show that they failed.

Modifications
-------------
 - Make AsyncState take an exception when Cancel or Complete is called
 - If the server fails to respond within the configured timeout, return a
   OperationTimeout.
 - If a operation is canceled, return an OperationAbandonedException for
   each operation.
 - Add OperationAbandonedException for canceled operations

 Results
 -------
 If a server connection times out, any in-flight operation will be
 canceled and an OperationAbandonedException will returned along with a
 status of ClientFailure.

Change-Id: I02af9d0134ac0b0c40a3c0aaf88278ddc8099dc3
Reviewed-on: http://review.couchbase.org/74124
Tested-by: Jeffry Morris <jeffrymorris@gmail.com>
Reviewed-by: Mike Goldsmith <goldsmith.mike@gmail.com>
  • Loading branch information
jeffrymorris committed Mar 1, 2017
1 parent 6242a3a commit ad23a3f
Show file tree
Hide file tree
Showing 9 changed files with 124 additions and 32 deletions.
5 changes: 4 additions & 1 deletion Src/Couchbase.NetStandard/Couchbase.NetStandard.csproj
Expand Up @@ -200,6 +200,9 @@
<Compile Include="..\Couchbase\Core\IO\SubDocument\OperationSpec.cs">
<Link>Core\IO\SubDocument\OperationSpec.cs</Link>
</Compile>
<Compile Include="..\Couchbase\IO\SendTimeoutExpiredException.cs">
<Link>IO\SendTimeoutExpiredException.cs</Link>
</Compile>
<Compile Include="..\Couchbase\Logging\CommonLoggingLogger.cs">
<Link>Logging\CommonLoggingLogger.cs</Link>
</Compile>
Expand Down Expand Up @@ -1357,4 +1360,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
</Project>
8 changes: 4 additions & 4 deletions Src/Couchbase.UnitTests/Couchbase.UnitTests.csproj
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
Expand Down Expand Up @@ -81,7 +81,7 @@
<Compile Include="Configuration\CouchbaseConfigContextTests.cs" />
<Compile Include="Configuration\Server\Providers\CarrierPublication\CarrierPublicationProviderTests.cs" />
<Compile Include="Configuration\Server\Providers\ConfigProviderBaseTests.cs" />
<Compile Include="Core\Buckets\KetamaKeyMapperTests.cs" />
<Compile Include="Core\Buckets\KetamaKeyMapperTests.cs" />
<Compile Include="Core\Transcoders\BinaryToJsonTranscoderTests.cs" />
<Compile Include="GlobalSetup.cs" />
<Compile Include="IO\ConnectionBaseTests.cs" />
Expand All @@ -90,7 +90,7 @@
<Compile Include="IO\Operations\AddTests.cs" />
<Compile Include="IO\Operations\AppendTests.cs" />
<Compile Include="IO\Operations\Authentication\SaslAuthenticationTests.cs" />
<Compile Include="IO\Operations\GetLTests.cs" />
<Compile Include="IO\Operations\GetLTests.cs" />
<Compile Include="IO\Operations\GetTests.cs" />
<Compile Include="IO\Operations\GetTTests.cs" />
<Compile Include="IO\Operations\HelloTests.cs" />
Expand Down Expand Up @@ -173,7 +173,7 @@
<EmbeddedResource Include="Data\with_escaped_quotes.json" />
<EmbeddedResource Include="Data\view_result.json" />
<EmbeddedResource Include="Data\view_result_with_reduce.json" />
<EmbeddedResource Include="Data\ketama-ring-hashes.json" />
<EmbeddedResource Include="Data\ketama-ring-hashes.json" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
Expand Down
37 changes: 19 additions & 18 deletions Src/Couchbase/Couchbase.csproj
Expand Up @@ -70,12 +70,12 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Analytics\AnalyticsRequest.cs" />
<Compile Include="Analytics\IAnalyticsClient.cs" />
<Compile Include="Analytics\IAnalyticsRequest.cs" />
<Compile Include="Analytics\IAnalyticsResult.cs" />
<Compile Include="Analytics\AnalyticsClient.cs" />
<Compile Include="Analytics\AnalyticsResult.cs" />
<Compile Include="Analytics\AnalyticsRequest.cs" />
<Compile Include="Analytics\IAnalyticsClient.cs" />
<Compile Include="Analytics\IAnalyticsRequest.cs" />
<Compile Include="Analytics\IAnalyticsResult.cs" />
<Compile Include="Analytics\AnalyticsClient.cs" />
<Compile Include="Analytics\AnalyticsResult.cs" />
<Compile Include="Authentication\AuthContext.cs" />
<Compile Include="Authentication\AuthType.cs" />
<Compile Include="Authentication\ClusterCredentials.cs" />
Expand Down Expand Up @@ -125,7 +125,7 @@
<Compile Include="Core\Buckets\KeySeqnoObserver.cs" />
<Compile Include="Core\Buckets\MutationToken.cs" />
<Compile Include="Core\ExpressionVisitors\LambdaSimplifyingExpressionVisitor.cs" />
<Compile Include="Core\FeatureNotAvailableException.cs" />
<Compile Include="Core\FeatureNotAvailableException.cs" />
<Compile Include="Core\ILookupInBuilder.cs" />
<Compile Include="Core\IMutateInBuilder.cs" />
<Compile Include="Core\IO\SubDocument\ISubdocInvoker.cs" />
Expand Down Expand Up @@ -242,11 +242,11 @@
<Compile Include="IO\Http\CouchbaseHttpClient.cs" />
<Compile Include="IO\Http\NonAuthenticatingHttpClientHandler.cs" />
<Compile Include="IO\IOBuffer.cs" />
<Compile Include="Core\SubdocLookupFlags.cs" />
<Compile Include="Core\SubdocLookupFlags.cs" />
<Compile Include="IO\Operations\SubDocument\MultiLookup.cs" />
<Compile Include="IO\Operations\SubDocument\MultiMutation.cs" />
<Compile Include="IO\Operations\SubDocument\SubDocDictUpsert.cs" />
<Compile Include="Core\SubdocMutateFlags.cs" />
<Compile Include="Core\SubdocMutateFlags.cs" />
<Compile Include="IO\Operations\SubDocument\SubDocReplace.cs" />
<Compile Include="IO\Operations\SubDocument\SubDocDictAdd.cs" />
<Compile Include="IO\Operations\SubDocument\SubDocDelete.cs" />
Expand All @@ -263,6 +263,7 @@
<Compile Include="IDocumentFragment%27.cs" />
<Compile Include="IO\ConnectionPoolFactory.cs" />
<Compile Include="IO\RemoteHostClosedException.cs" />
<Compile Include="IO\SendTimeoutExpiredException.cs" />
<Compile Include="IQueryCacheInvalidator.cs" />
<Compile Include="IDocumentResult.cs" />
<Compile Include="IO\AsyncState.cs" />
Expand Down Expand Up @@ -292,11 +293,11 @@
<Compile Include="IO\Operations\Unlock.cs" />
<Compile Include="IO\RemoteHostTimeoutException.cs" />
<Compile Include="IO\SocketAsyncState.cs" />
<Compile Include="Logging\CommonLoggingLogger.cs" />
<Compile Include="Logging\ILog.cs" />
<Compile Include="Logging\LogLevel.cs" />
<Compile Include="Logging\LogManager.cs" />
<Compile Include="Logging\MicrosoftLoggingLogger.cs" />
<Compile Include="Logging\CommonLoggingLogger.cs" />
<Compile Include="Logging\ILog.cs" />
<Compile Include="Logging\LogLevel.cs" />
<Compile Include="Logging\LogManager.cs" />
<Compile Include="Logging\MicrosoftLoggingLogger.cs" />
<Compile Include="Management\BucketSettings.cs" />
<Compile Include="Management\ClusterProvisioner.cs" />
<Compile Include="Management\ClusterSettings.cs" />
Expand Down Expand Up @@ -492,10 +493,10 @@
<Compile Include="Utils\UnsupportedAddressFamilyException.cs" />
<Compile Include="Utils\UriExtensions.cs" />
<Compile Include="Views\SpatialViewQuery.cs" />
<Compile Include="StreamAlreadyReadException.cs" />
<Compile Include="Views\StreamingViewClient.cs" />
<Compile Include="Views\StreamingViewResult.cs" />
<Compile Include="Views\ViewClientBase.cs" />
<Compile Include="StreamAlreadyReadException.cs" />
<Compile Include="Views\StreamingViewClient.cs" />
<Compile Include="Views\StreamingViewResult.cs" />
<Compile Include="Views\ViewClientBase.cs" />
<Compile Include="Views\ViewResult%27.cs" />
<Compile Include="Views\Error.cs" />
<Compile Include="Views\IDataMapper.cs" />
Expand Down
17 changes: 14 additions & 3 deletions Src/Couchbase/IO/AsyncState.cs
Expand Up @@ -20,17 +20,20 @@ internal class AsyncState : IState
/// <summary>
/// Cancels the current Memcached request that is in-flight.
/// </summary>
public void Cancel()
public void Cancel(ResponseStatus status, Exception e = null)
{
Timer.Dispose();

var response = new byte[24];
Converter.FromUInt32(Id, response, HeaderIndexFor.Opaque);

Callback(new SocketAsyncState
{
Data = new MemoryStream(response),
Opaque = Id,
Status = ResponseStatus.ClientFailure
// ReSharper disable once MergeConditionalExpression
Exception = e,
Status = status
});
}

Expand All @@ -42,18 +45,26 @@ public void Complete(byte[] response)
{
Timer.Dispose();

//defaults
var status = ResponseStatus.None;
Exception e = null;

//this means the request never completed
if (response == null)
{
response = new byte[24];
Converter.FromUInt32(Id, response, HeaderIndexFor.Opaque);
e = new SendTimeoutExpiredException();
status = ResponseStatus.TransportFailure;
}

//somewhat of hack for backwards compatibility
Callback(new SocketAsyncState
{
Data = new MemoryStream(response),
Opaque = Id
Opaque = Id,
Exception = e,
Status = status
});
}
}
Expand Down
4 changes: 2 additions & 2 deletions Src/Couchbase/IO/IState.cs
@@ -1,12 +1,12 @@

using System;

namespace Couchbase.IO
{
/// <summary>
/// Represents a Memcached request in flight.
/// </summary>
internal interface IState
{

/// <summary>
/// Completes the specified Memcached response.
/// </summary>
Expand Down
5 changes: 3 additions & 2 deletions Src/Couchbase/IO/MultiplexingConnection.cs
Expand Up @@ -73,7 +73,7 @@ public override void SendAsync(byte[] request, Func<SocketAsyncState, Task> call
IState inflight;
_statesInFlight.TryRemove(a.Id, out inflight);
}
a.Cancel();
a.Cancel(ResponseStatus.OperationTimeout, new SendTimeoutExpiredException());
}, state, Configuration.SendTimeout, Timeout.Infinite);

var sentBytesCount = 0;
Expand Down Expand Up @@ -143,7 +143,7 @@ public override byte[] Send(byte[] request)

if (!didComplete)
{
throw new TimeoutException();
throw new SendTimeoutExpiredException();
}

return response;
Expand Down Expand Up @@ -276,6 +276,7 @@ private void HandleDisconnect(Exception exception)
//in any case the current socket object should be closed, all states in flight released etc.
Close();
}

public void Close()
{
if (Disposed) return;
Expand Down
65 changes: 65 additions & 0 deletions Src/Couchbase/IO/SendTimeoutExpiredException.cs
@@ -0,0 +1,65 @@
using System;
using Couchbase.Configuration.Client;
using Couchbase.Utils;

namespace Couchbase.IO
{
/// <summary>
/// Thrown if an operation does not complete before the <see cref="PoolConfiguration.SendTimeout"/> is exceeded.
/// </summary>
/// <seealso cref="System.TimeoutException" />
public sealed class SendTimeoutExpiredException : TimeoutException
{
private string _stackTrace;

public SendTimeoutExpiredException()
: this(ExceptionUtil.GetMessage(ExceptionUtil.OperationTimeout))
{
}

public SendTimeoutExpiredException(string message)
: base(message)
{
_stackTrace = Environment.StackTrace;
Source = CurrentAssembly.Current.GetName().Name;
}

public SendTimeoutExpiredException(string message, Exception innerException)
: base(message, innerException)
{
_stackTrace = Environment.StackTrace;
Source = CurrentAssembly.Current.GetName().Name;
}

/// <summary>
/// Gets a string representation of the immediate frames on the call stack.
/// </summary>
public override string StackTrace
{
get { return _stackTrace; }
}
}
}

#region [ License information ]

/* ************************************************************
*
* @author Couchbase <info@couchbase.com>
* @copyright 2015 Couchbase, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* ************************************************************/

#endregion
13 changes: 11 additions & 2 deletions Src/Couchbase/IO/SyncState.cs
@@ -1,14 +1,23 @@

using System;
using System.Threading;
using System.Xml.Serialization;


namespace Couchbase.IO
{
/// <summary>
/// Represents a synchronous Memcached operation.
/// </summary>
/// <seealso cref="Couchbase.IO.IState" />
internal class SyncState : IState
{
public byte[] Response;
public readonly AutoResetEvent SyncWait = new AutoResetEvent(false);

/// <summary>
/// Completes the specified Memcached response.
/// </summary>
/// <param name="response">The Memcached response packet.</param>
/// <remarks>Exception is not used</remarks>
public void Complete(byte[] response)
{
Response = response;
Expand Down
2 changes: 2 additions & 0 deletions Src/Couchbase/Utils/ExceptionUtil.cs
Expand Up @@ -76,6 +76,8 @@ public static class ExceptionUtil

public const string ParameterCannotBeNullOrEmptyFormat = "{0} cannot be null, empty or whitespace.";

public const string OperationTimeout = "The operation has timed out.";

public static string GetNodeUnavailableMsg(IPEndPoint ipEndPoint, uint interval)
{
return string.Format(NodeUnavailableMsg, ipEndPoint, interval);
Expand Down

0 comments on commit ad23a3f

Please sign in to comment.