Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#14 Erroneous TeamCity service messages with multiple test assemblies and (default) parallel execution #18

Merged
merged 3 commits into from
Sep 12, 2016
+680 −170
Merged

#14 Erroneous TeamCity service messages with multiple test assemblies and (default) parallel execution #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
39f7e18
#14 Erroneous TeamCity service messages with multiple test assemblie…
Sep 8, 2016
01fdcd5
#14 Erroneous TeamCity service messages with multiple test assemblies…
Sep 9, 2016
ae3f55d
#14 Erroneous TeamCity service messages with multiple test assemblies…
Sep 9, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 38 additions & 16 deletions src/extension/TeamCityEventListener.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,13 @@ public class TeamCityEventListener : ITestEventListener
private static readonly ServiceMessageWriter ServiceMessageWriter = new ServiceMessageWriter();
private readonly TextWriter _outWriter;
private readonly Dictionary<string, string> _refs = new Dictionary<string, string>();
private int _blockCounter;
private string _rootFlowId;
private readonly Dictionary<string, int> _blockCounters = new Dictionary<string, int>();

public TeamCityEventListener() : this(Console.Out) { }

public TeamCityEventListener(TextWriter outWriter)
{
if (outWriter == null) throw new ArgumentNullException("outWriter");
if (outWriter == null) throw new ArgumentNullException("outWriter");

_outWriter = outWriter;
}
Expand Down Expand Up @@ -93,8 +92,13 @@ public void RegisterMessage(XmlNode testEvent)
}

var id = testEvent.GetAttribute("id");
if (id == null)
{
id = string.Empty;
}

var parentId = testEvent.GetAttribute("parentId");
string flowId;
var flowId = ".";
if (parentId != null)
{
// NUnit 3 case
Expand All @@ -104,7 +108,14 @@ public void RegisterMessage(XmlNode testEvent)
else
{
// NUnit 2 case
flowId = _rootFlowId;
if (!string.IsNullOrEmpty(id))
{
var idParts = id.Split('-');
if (idParts.Length == 2)
{
flowId = idParts[0];
}
}
}

string testFlowId;
Expand All @@ -125,12 +136,12 @@ public void RegisterMessage(XmlNode testEvent)
{
case "start-suite":
_refs[id] = parentId;
StartSuiteCase(id, parentId, flowId, fullName);
StartSuiteCase(parentId, flowId, fullName);
break;

case "test-suite":
_refs.Remove(id);
TestSuiteCase(id, parentId, flowId, fullName);
TestSuiteCase(parentId, flowId, fullName);
break;

case "start-test":
Expand Down Expand Up @@ -194,7 +205,7 @@ private void CaseStartTest(string id, string flowId, string parentId, string tes
OnTestStart(testFlowId, fullName);
}

private void TestSuiteCase(string id, string parentId, string flowId, string fullName)
private void TestSuiteCase(string parentId, string flowId, string fullName)
{
// NUnit 3 case
if (parentId == string.Empty)
Expand All @@ -205,15 +216,14 @@ private void TestSuiteCase(string id, string parentId, string flowId, string ful
// NUnit 2 case
if (parentId == null)
{
if (--_blockCounter == 0)
if (ChangeBlockCounter(flowId, -1) == 0)
{
_rootFlowId = null;
OnRootSuiteFinish(id, fullName);
OnRootSuiteFinish(flowId, fullName);
}
}
}

private void StartSuiteCase(string id, string parentId, string flowId, string fullName)
private void StartSuiteCase(string parentId, string flowId, string fullName)
{
// NUnit 3 case
if (parentId == string.Empty)
Expand All @@ -224,14 +234,26 @@ private void StartSuiteCase(string id, string parentId, string flowId, string fu
// NUnit 2 case
if (parentId == null)
{
if (_blockCounter++ == 0)
if (ChangeBlockCounter(flowId, 1) == 1)
{
_rootFlowId = id;
OnRootSuiteStart(id, fullName);
OnRootSuiteStart(flowId, fullName);
}
}
}

private int ChangeBlockCounter(string flowId, int changeValue)
{
int currentBlockCounter;
if (!_blockCounters.TryGetValue(flowId, out currentBlockCounter))
{
currentBlockCounter = 0;
}

currentBlockCounter += changeValue;
_blockCounters[flowId] = currentBlockCounter;
return currentBlockCounter;
}

private bool TryFindParentId(string id, out string parentId)
{
if (id == null)
Expand Down Expand Up @@ -278,7 +300,7 @@ private void TrySendOutput(string flowId, XmlNode message, string fullName)
new ServiceMessageAttr(ServiceMessageAttr.Names.Name, fullName),
new ServiceMessageAttr(ServiceMessageAttr.Names.Out, outputStr),
new ServiceMessageAttr(ServiceMessageAttr.Names.FlowId, flowId),
new ServiceMessageAttr(ServiceMessageAttr.Names.TcTags, "tc:parseServiceMessagesInside")));
new ServiceMessageAttr(ServiceMessageAttr.Names.TcTags, "tc:parseServiceMessagesInside")));
}

private void OnRootSuiteStart(string flowId, string assemblyName)
Expand Down
2 changes: 1 addition & 1 deletion src/extension/teamcity-event-listener.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>..\..\bin\nunit\NUnit.Extension.TeamCityEventListener\tools\</OutputPath>
<OutputPath>..\..\..\nunit-console\bin\Debug\addins\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
Expand Down
129 changes: 114 additions & 15 deletions src/nunit.integration.tests/RunTests.feature
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,119 @@
Background:
Given NUnit path is ..\nunit\

Scenario Outline: User runs tests for several assemblies
Given Framework version is <frameworkVersion>
And I have created the folder mocks
And I have copied NUnit framework references to folder mocks
And I have added successful method as SuccessfulTest to the class Foo.Tests.UnitTests1 for foo.tests
And I have added NUnit framework references to foo.tests

And I have compiled the assembly foo.tests to file mocks\foo1.tests.dll
And I have added the assembly mocks\foo1.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo2.tests.dll
And I have added the assembly mocks\foo2.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo3.tests.dll
And I have added the assembly mocks\foo3.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo4.tests.dll
And I have added the assembly mocks\foo4.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo5.tests.dll
And I have added the assembly mocks\foo5.tests.dll to the list of testing assemblies

And I have compiled the assembly foo.tests to file mocks\foo6.tests.dll
And I have added the assembly mocks\foo6.tests.dll to the list of testing assemblies

And I want to use <configurationType> configuration type
And I have added the arg workers=10 to NUnit console command line
And I have added the arg agents=<agents> to NUnit console command line
And I have added the arg process=<process> to NUnit console command line
And I have added the arg domain=<domain> to NUnit console command line

When I run NUnit console
Then the exit code should be 0
And the output should contain correct set of TeamCity service messages
And the Test Run Summary should has following:
| field | value |
| Test Count | 6 |
| Passed | 6 |
| Failed | 0 |
| Inconclusive | 0 |
| Skipped | 0 |

Examples:
| frameworkVersion | process | domain | agents | platform | configurationType |
| Version40 | InProcess | Single | 2 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 2 | AnyCpu | ProjectFile |
| Version40 | Separate | Single | 2 | AnyCpu | ProjectFile |
| Version45 | Multiple | Single | 2 | AnyCpu | ProjectFile |
| Version40 | Multiple | Single | 2 | AnyCpu | ProjectFile |
| Version45 | InProcess | Multiple | 2 | AnyCpu | ProjectFile |
| Version40 | InProcess | Multiple | 2 | AnyCpu | ProjectFile |
| Version45 | Separate | Multiple | 2 | AnyCpu | ProjectFile |
| Version40 | Separate | Multiple | 2 | AnyCpu | ProjectFile |
| Version45 | InProcess | Single | 1 | AnyCpu | ProjectFile |
| Version40 | InProcess | Single | 1 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 1 | AnyCpu | ProjectFile |
| Version40 | Separate | Single | 1 | AnyCpu | ProjectFile |
| Version45 | Multiple | Single | 1 | AnyCpu | ProjectFile |
| Version40 | Multiple | Single | 1 | AnyCpu | ProjectFile |
| Version45 | InProcess | Multiple | 1 | AnyCpu | ProjectFile |
| Version40 | InProcess | Multiple | 1 | AnyCpu | ProjectFile |
| Version45 | Separate | Multiple | 1 | AnyCpu | ProjectFile |
| Version40 | Separate | Multiple | 1 | AnyCpu | ProjectFile |
| Version45 | Separate | Single | 1 | X86 | ProjectFile |
| Version40 | Separate | Single | 1 | X86 | ProjectFile |
| Version45 | Multiple | Single | 1 | X86 | ProjectFile |
| Version40 | Multiple | Single | 1 | X86 | ProjectFile |
| Version45 | Separate | Multiple | 1 | X86 | ProjectFile |
| Version40 | Separate | Multiple | 1 | X86 | ProjectFile |
| Version40 | InProcess | Single | 2 | X86 | ProjectFile |
| Version45 | Separate | Single | 2 | X86 | ProjectFile |
| Version40 | Separate | Single | 2 | X86 | ProjectFile |
| Version45 | Multiple | Single | 2 | X86 | ProjectFile |
| Version40 | Multiple | Single | 2 | X86 | ProjectFile |
| Version45 | InProcess | Multiple | 2 | X86 | ProjectFile |
| Version40 | InProcess | Multiple | 2 | X86 | ProjectFile |
| Version45 | Separate | Multiple | 2 | X86 | ProjectFile |
| Version40 | Separate | Multiple | 2 | X86 | ProjectFile |
| Version40 | InProcess | Single | 2 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 2 | AnyCpu | CmdArguments |
| Version40 | Separate | Single | 2 | AnyCpu | CmdArguments |
| Version45 | Multiple | Single | 2 | AnyCpu | CmdArguments |
| Version40 | Multiple | Single | 2 | AnyCpu | CmdArguments |
| Version45 | InProcess | Multiple | 2 | AnyCpu | CmdArguments |
| Version40 | InProcess | Multiple | 2 | AnyCpu | CmdArguments |
| Version45 | Separate | Multiple | 2 | AnyCpu | CmdArguments |
| Version40 | Separate | Multiple | 2 | AnyCpu | CmdArguments |
| Version45 | InProcess | Single | 1 | AnyCpu | CmdArguments |
| Version40 | InProcess | Single | 1 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 1 | AnyCpu | CmdArguments |
| Version40 | Separate | Single | 1 | AnyCpu | CmdArguments |
| Version45 | Multiple | Single | 1 | AnyCpu | CmdArguments |
| Version40 | Multiple | Single | 1 | AnyCpu | CmdArguments |
| Version45 | InProcess | Multiple | 1 | AnyCpu | CmdArguments |
| Version40 | InProcess | Multiple | 1 | AnyCpu | CmdArguments |
| Version45 | Separate | Multiple | 1 | AnyCpu | CmdArguments |
| Version40 | Separate | Multiple | 1 | AnyCpu | CmdArguments |
| Version45 | Separate | Single | 1 | X86 | CmdArguments |
| Version40 | Separate | Single | 1 | X86 | CmdArguments |
| Version45 | Multiple | Single | 1 | X86 | CmdArguments |
| Version40 | Multiple | Single | 1 | X86 | CmdArguments |
| Version45 | Separate | Multiple | 1 | X86 | CmdArguments |
| Version40 | Separate | Multiple | 1 | X86 | CmdArguments |
| Version40 | InProcess | Single | 2 | X86 | CmdArguments |
| Version45 | Separate | Single | 2 | X86 | CmdArguments |
| Version40 | Separate | Single | 2 | X86 | CmdArguments |
| Version45 | Multiple | Single | 2 | X86 | CmdArguments |
| Version40 | Multiple | Single | 2 | X86 | CmdArguments |
| Version45 | InProcess | Multiple | 2 | X86 | CmdArguments |
| Version40 | InProcess | Multiple | 2 | X86 | CmdArguments |
| Version45 | Separate | Multiple | 2 | X86 | CmdArguments |
| Version40 | Separate | Multiple | 2 | X86 | CmdArguments |

Scenario Outline: User runs parallelizable tests
Given Framework version is <frameworkVersion>
And I have added SuccessfulParallelizable method as SuccessfulParallelizable1 to the class Foo.Tests.UnitTests1 for foo1.tests
Expand Down Expand Up @@ -53,8 +166,6 @@ Examples:
| Version40 | InProcess | Multiple | 10 | AnyCpu |
| Version45 | Separate | Multiple | 10 | AnyCpu |
| Version40 | Separate | Multiple | 10 | AnyCpu |
# | Version45 | Multiple | Multiple | 10 | AnyCpu |
# | Version40 | Multiple | Multiple | 10 | AnyCpu |
| Version45 | InProcess | Single | 1 | AnyCpu |
| Version40 | InProcess | Single | 1 | AnyCpu |
| Version45 | Separate | Single | 1 | AnyCpu |
Expand All @@ -65,24 +176,18 @@ Examples:
| Version40 | InProcess | Multiple | 1 | AnyCpu |
| Version45 | Separate | Multiple | 1 | AnyCpu |
| Version40 | Separate | Multiple | 1 | AnyCpu |
# | Version45 | Multiple | Multiple | 1 | AnyCpu |
# | Version40 | Multiple | Multiple | 1 | AnyCpu |
| Version45 | Separate | Single | 10 | X86 |
| Version40 | Separate | Single | 10 | X86 |
| Version45 | Multiple | Single | 10 | X86 |
| Version40 | Multiple | Single | 10 | X86 |
| Version45 | Separate | Multiple | 10 | X86 |
| Version40 | Separate | Multiple | 10 | X86 |
# | Version45 | Multiple | Multiple | 10 | X86 |
# | Version40 | Multiple | Multiple | 10 | X86 |
| Version45 | Separate | Single | 1 | X86 |
| Version40 | Separate | Single | 1 | X86 |
| Version45 | Multiple | Single | 1 | X86 |
| Version40 | Multiple | Single | 1 | X86 |
| Version45 | Separate | Multiple | 1 | X86 |
| Version40 | Separate | Multiple | 1 | X86 |
# | Version45 | Multiple | Multiple | 1 | X86 |
# | Version40 | Multiple | Multiple | 1 | X86 |

Scenario Outline: User runs parallelizable tests for NUnit 2 framework
And I have added successful method as SuccessfulTest to the class Foo.Tests.UnitTests1 for foo.tests
Expand Down Expand Up @@ -134,21 +239,15 @@ Examples:
| Version40 | InProcess | Multiple | 1 | AnyCpu |
| Version45 | Separate | Multiple | 1 | AnyCpu |
| Version40 | Separate | Multiple | 1 | AnyCpu |
# | Version45 | Multiple | Multiple | 1 | AnyCpu |
# | Version40 | Multiple | Multiple | 1 | AnyCpu |
| Version45 | Separate | Single | 10 | X86 |
| Version40 | Separate | Single | 10 | X86 |
| Version45 | Multiple | Single | 10 | X86 |
| Version40 | Multiple | Single | 10 | X86 |
| Version45 | Separate | Multiple | 10 | X86 |
| Version40 | Separate | Multiple | 10 | X86 |
# | Version45 | Multiple | Multiple | 10 | X86 |
# | Version40 | Multiple | Multiple | 10 | X86 |
| Version45 | Separate | Single | 1 | X86 |
| Version40 | Separate | Single | 1 | X86 |
| Version45 | Multiple | Single | 1 | X86 |
| Version40 | Multiple | Single | 1 | X86 |
| Version45 | Separate | Multiple | 1 | X86 |
| Version40 | Separate | Multiple | 1 | X86 |
# | Version45 | Multiple | Multiple | 1 | X86 |
# | Version40 | Multiple | Multiple | 1 | X86 |
| Version40 | Separate | Multiple | 1 | X86 |
Loading