Special test scenario with new unit test framework fails #422

taori · 2020-01-17T20:39:30Z

I've got a scenario where my test case has an error and a warning - i provide a codefix or the warning. After the warning is fixed, the error is fixed automatically too.

If i expect both diagnostics get this:

If i expect just one diagnostic i get this:

The old Inherited CodeFixTest version makes the test pass just fine.

this is my test method:

		[TestMethod]
		public async Task MixedMethodsWithPartialFix()
		{
			var test = @"namespace ConsoleApplication1
{
	using System;
	using System.Collections.Generic;
	using System.Linq;
	using System.Text;
	using System.Threading.Tasks;
	using System.Diagnostics;
	using System.IO;

	public class TestClass
	{
		private string _workspaceFile;

		public async Task<Configuration[]> LoadStorageContentAsync()
		{
			var fileInfo = new FileInfo(_workspaceFile);
			if (!fileInfo.Exists)
			{
				Log.Error($""No configuration storage located at { fileInfo.FullName}."");
				return Array.Empty<Configuration>();
			}

			using (var stream = new StreamReader(new FileStream(fileInfo.FullName, FileMode.Open)))
			{
				try
				{
					var storage = new Storage();
					return Task.FromResult(storage.Configurations.ToArray());
				}
				catch (Exception e)
				{
					Log.Error(e);
					return Array.Empty<Configuration>();
				}
			}
		}

		public class Log
		{
			public static void Error(Exception p0)
			{
				throw new NotImplementedException();
			}

			public static void Error(string p0)
			{
				throw new NotImplementedException();
			}
		}

		public class Storage
		{
			public Configuration[] Configurations { get; set; }
		}
		public class Configuration
		{
			public Guid Id { get; set; }
			public string ConfigurationName { get; set; }
		}
	}
}";


			var fixtest = @"namespace ConsoleApplication1
{
	using System;
	using System.Collections.Generic;
	using System.Linq;
	using System.Text;
	using System.Threading.Tasks;
	using System.Diagnostics;
	using System.IO;

	public class TestClass
	{
		private string _workspaceFile;

		public Task<Configuration[]> LoadStorageContentAsync()
		{
			var fileInfo = new FileInfo(_workspaceFile);
			if (!fileInfo.Exists)
			{
				Log.Error($""No configuration storage located at { fileInfo.FullName}."");
				return Task.FromResult(Array.Empty<Configuration>());
			}

			using (var stream = new StreamReader(new FileStream(fileInfo.FullName, FileMode.Open)))
			{
				try
				{
					var storage = new Storage();
					return Task.FromResult(storage.Configurations.ToArray());
				}
				catch (Exception e)
				{
					Log.Error(e);
					return Task.FromResult(Array.Empty<Configuration>());
				}
			}
		}

		public class Log
		{
			public static void Error(Exception p0)
			{
				throw new NotImplementedException();
			}

			public static void Error(string p0)
			{
				throw new NotImplementedException();
			}
		}

		public class Storage
		{
			public Configuration[] Configurations { get; set; }
		}
		public class Configuration
		{
			public Guid Id { get; set; }
			public string ConfigurationName { get; set; }
		}
	}
}";

			// await new CodeFixTest<EmptyDiagnosticAnalyzer,
			// 	Amusoft.CodeAnalysis.Analyzers.CS1998.FixByWrappingInTaskResult>()
			// {
			// 	CompilerDiagnostics = CompilerDiagnostics.Errors | CompilerDiagnostics.Warnings,
			// 	TestState =
			// 	{
			// 		Sources = { test },
			// 		ExpectedDiagnostics =
			// 		{
			// 			CompilerWarning("CS1591").WithSpan(11, 15, 11, 24),
			// 			CompilerWarning("CS0649").WithSpan(13, 18, 13, 32),
			// 			CompilerWarning("CS1591").WithSpan(15, 38, 15, 61),
			// 			CompilerWarning("CS1998").WithLocation(15,38),
			// 			CompilerError("CS4016").WithSpan(29, 13, 29, 62).WithArguments("ConsoleApplication1.TestClass.Configuration[]"),
			//
			// 			CompilerWarning("CS1591").WithSpan(39, 16, 39, 19),
			// 			CompilerWarning("CS1591").WithSpan(41, 23, 41, 28),
			// 			CompilerWarning("CS1591").WithSpan(46, 23, 46, 28),
			// 			CompilerWarning("CS1591").WithSpan(52, 16, 52, 23),
			// 			CompilerWarning("CS1591").WithSpan(54, 27, 54, 41),
			// 			CompilerWarning("CS1591").WithSpan(56, 16, 56, 29),
			// 			CompilerWarning("CS1591").WithSpan(58, 16, 58, 18),
			// 			CompilerWarning("CS1591").WithSpan(59, 18, 59, 35),
			// 		},
			// 	},
			// 	FixedState =
			// 	{
			// 		ExpectedDiagnostics =
			// 		{
			// 			CompilerWarning("CS1591").WithSpan(11, 15, 11, 24),
			// 			CompilerWarning("CS0649").WithSpan(13, 18, 13, 32),
			// 			CompilerWarning("CS1591").WithSpan(15, 32, 15, 55),
			// 			CompilerWarning("CS1591").WithSpan(39, 16, 39, 19),
			// 			CompilerWarning("CS1591").WithSpan(41, 23, 41, 28),
			// 			CompilerWarning("CS1591").WithSpan(46, 23, 46, 28),
			// 			CompilerWarning("CS1591").WithSpan(52, 16, 52, 23),
			// 			CompilerWarning("CS1591").WithSpan(54, 27, 54, 41),
			// 			CompilerWarning("CS1591").WithSpan(56, 16, 56, 29),
			// 			CompilerWarning("CS1591").WithSpan(58, 16, 58, 18),
			// 			CompilerWarning("CS1591").WithSpan(59, 18, 59, 35),
			// 		},
			// 		Sources = { fixtest }
			// 	},
			// }.RunAsync();
			var expected = new[]
			{
				CompilerWarning("CS1998").WithLocation(15, 38),
				// CompilerError("CS4016").WithSpan(29, 13, 29, 62)
				// 	.WithArguments("ConsoleApplication1.TestClass.Configuration[]"),
			};

			await Verifier.VerifyCodeFixAsync(test, expected, fixtest);
		}

This is my CodeFix:

	[ExportCodeFixProvider(LanguageNames.CSharp, Name = "CS1998-FixByWrappingInTaskResult"), Shared]
	public class FixByWrappingInTaskResult : CodeFixProviderBase
	{
		/// <inheritdoc />
		protected override string DiagnosticId { get; } = "CS1998";

		/// <inheritdoc />
		protected override string GetEquivalenceKey(SyntaxNode rootNode)
		{
			return "CS1998-FixByWrappingInTaskResult";
		}

		/// <inheritdoc />
		protected override string GetTitle(SyntaxNode rootNode)
		{
			return Resources.MessageFormat_CS1998_FixByWrappingInTaskResult;
		}

		/// <inheritdoc />
		protected override async Task<SyntaxNode> FixedDiagnosticAsync(SyntaxNode rootNode, SyntaxNode diagnosticNode,
			CodeFixContext context,
			CancellationToken cancellationToken)
		{
			var semanticModel = await context.Document.GetSemanticModelAsync(cancellationToken)
				.ConfigureAwait(false);
			if (diagnosticNode is MethodDeclarationSyntax methodDeclarationSyntax)
			{
				var controlFlowAnalysis = semanticModel.AnalyzeControlFlow(methodDeclarationSyntax.Body);
				if (controlFlowAnalysis.ReturnStatements.Length > 0)
				{
					var documentEditor = await DocumentEditor.CreateAsync(context.Document, cancellationToken)
						.ConfigureAwait(false);

					RemoveAsyncFromMethod(documentEditor, methodDeclarationSyntax, semanticModel);

					foreach (var returnStatement in controlFlowAnalysis.ReturnStatements)
					{
						if (returnStatement is ReturnStatementSyntax returnStatementSyntax)
						{
							if (!ShouldRewrite(returnStatementSyntax))
								continue;

							documentEditor.ReplaceNode(returnStatementSyntax, RewriteExit(returnStatementSyntax));
						}
					}

					return await documentEditor.GetChangedDocument().GetSyntaxRootAsync(cancellationToken);
				}
			}

			return rootNode;
		}

		private static void RemoveAsyncFromMethod(DocumentEditor documentEditor, MethodDeclarationSyntax methodDeclarationSyntax, SemanticModel semanticModel)
		{
			documentEditor.SetModifiers(methodDeclarationSyntax, DeclarationModifiers.From(semanticModel.GetDeclaredSymbol(methodDeclarationSyntax)).WithAsync(false));
		}

		private bool ShouldRewrite(ReturnStatementSyntax returnStatementSyntax)
		{
			if (returnStatementSyntax.Expression is InvocationExpressionSyntax invocationExpressionSyntax)
			{
				if (invocationExpressionSyntax.Expression is MemberAccessExpressionSyntax memberAccessExpressionSyntax)
				{
					if (memberAccessExpressionSyntax.Name.Identifier.Text.Equals("FromResult")
					&& memberAccessExpressionSyntax.Expression is IdentifierNameSyntax expressionIdentifierName
					&& expressionIdentifierName.Identifier.Text.Equals("Task"))
					{
						return false;
					}
				}
			}

			return true;
		}

		private SyntaxNode RewriteExit(ReturnStatementSyntax returnStatement)
		{
			return returnStatement.WithExpression(
				InvocationExpression(
					MemberAccessExpression(
						SyntaxKind.SimpleMemberAccessExpression,
						IdentifierName("Task"),
						IdentifierName("FromResult")))
				.WithArgumentList(
					ArgumentList(
						SingletonSeparatedList<ArgumentSyntax>(
							Argument(
								returnStatement.Expression))))
				);
		}
	}

	/// <summary>
	/// <inheritdoc />
	/// </summary>
	public abstract class CodeFixProviderBase : CodeFixProvider
	{
		public sealed override FixAllProvider GetFixAllProvider()
		{
			// See https://github.com/dotnet/roslyn/blob/master/docs/analyzers/FixAllProvider.md for more information on Fix All Providers
			return WellKnownFixAllProviders.BatchFixer;
		}

		/// <inheritdoc />
		public override ImmutableArray<string> FixableDiagnosticIds => new[] { DiagnosticId }.ToImmutableArray();

		protected abstract string DiagnosticId { get; }

		protected const string TitleAnnotation = "META:CodeFixTitleAnnotation";
		protected const string MemberAnnotation = "META:MemberAnnotation";
		protected const string TypeAnnotation = "META:TypeAnnotation";

		protected abstract string GetEquivalenceKey(SyntaxNode rootNode);
		protected abstract string GetTitle(SyntaxNode rootNode);

		protected virtual async Task<Document> GetFixedDocumentAsync(CodeFixContext context,
			CancellationToken cancellationToken, Diagnostic diagnostic)
		{
			var rootNode = await context.Document.GetSyntaxRootAsync(cancellationToken)
				.ConfigureAwait(false);
			var fixedRoot = await FixedDiagnosticAsync(rootNode, rootNode.FindNode(diagnostic.Location.SourceSpan), context, cancellationToken)
				.ConfigureAwait(false);

			return context.Document
				.WithSyntaxRoot(fixedRoot);
		}

		protected abstract Task<SyntaxNode> FixedDiagnosticAsync(SyntaxNode rootNode, SyntaxNode diagnosticNode,
			CodeFixContext context, CancellationToken cancellationToken);

		/// <inheritdoc />
		public override async Task RegisterCodeFixesAsync(CodeFixContext context)
		{
			var originalRoot = await context.Document.GetSyntaxRootAsync(context.CancellationToken)
				.ConfigureAwait(false);

			foreach (var diagnostic in context.Diagnostics)
			{
				if (!diagnostic.Id.Equals(DiagnosticId, StringComparison.OrdinalIgnoreCase))
					continue;

				var fixedDocument = await GetFixedDocumentAsync(context, context.CancellationToken, diagnostic)
					.ConfigureAwait(false);
				var fixedRoot = await fixedDocument.GetSyntaxRootAsync(context.CancellationToken)
					.ConfigureAwait(false);

				if(fixedRoot.IsEquivalentTo(originalRoot))
					continue;

				context.RegisterCodeFix(CodeAction.Create(GetTitle(fixedRoot), c => Task.FromResult(fixedDocument), GetEquivalenceKey(fixedRoot)), diagnostic);
			}
		}

		protected string GetAnnotationValue(SyntaxNode rootNode, string annotation, string defaultValue = "unknown annotation")
		{
			return rootNode.GetAnnotations(annotation).FirstOrDefault(d => d.Kind == annotation)?.Data ?? defaultValue;
		}
	}

@sharwell Willing to contribute again if this is an option. Unless you want to take care of it yourself of course :)

sharwell · 2020-01-17T21:09:31Z

This sounds like a case where it would be necessary to explicitly provide the expected diagnostics for the fixed state.

taori · 2020-01-17T21:22:02Z

Technically this problem will only arise when dealing with compiler diagnostics anyway, right? In that case it is just just more verbose to deal with advanced scenarios

sharwell · 2020-01-17T21:23:12Z

Not necessarily. It is also known to occur in a few other cases:

Testing multiple analyzers at once
Testing an analyzer that reports multiple different diagnostics

taori · 2020-01-17T22:08:23Z

Hm. Still getting unexpected behavior in my test method here: VerifyDiagnosticMethodFiltering

https://github.com/taori/Amusoft.CodeAnalysis.Analyzers/blob/f9f205644be5f2bf498aa911f4538c5c379a78f8/src/Amusoft.CodeAnalysis.Analyzers.Test/Tests/ACA0001/DetectionTests.cs#L273

neither one of those calls in my refactoring branch (refactoring-remove-old-testframework) work. The diagnostics related to my Analyzer seem to be excluded. Can you please spot my test for any errors?

For the simple syntax i get 0/4 diagnostics.
For the verbose syntax i get 8/12 diagnostics.

sharwell · 2020-01-17T23:11:45Z

The clue is here:

Context: Verifying exclusions in <auto-generated> code

The test is failing because the test framework added the following comment to the top of the input file, but the diagnostic failed to go away:

// <auto-generated/>

If your analyzer intends to report diagnostics in generated code, it needs to use the full form for testing and disable the test for generated code.

await new VerifyCS.Test {
  TestBehaviors = TestBehaviors.SkipGeneratedCodeCheck,
  // ...
}.RunAsync();

If the analyzer does not need to report diagnostics in generated code, then it should be configured in the Initialize method:

context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None);

taori · 2020-01-18T18:34:59Z

Thanks! that worked. What do you think about perhaps adding such remarks to the output of a test? In the past when working with dotnet tools i recently noticed how sometimes they include indicators of what settings to change if you run into edge cases which solve your problem?

sharwell · 2020-01-18T19:54:07Z

That seems like a good idea. It could either print the message directly, or it could link to a help page in the documentation for the test library.

taori · 2020-01-18T20:10:44Z

Combining both (while keeping the notice short) would be useful. Personally i was unaware of this configuration and anyone who is interested in learning about would have a place to read up on it.

taori · 2020-01-18T20:46:57Z

@sharwell what are your thoughts on introducing a settings property to alter the values of the test prior to running it - the way it is right now i have to use a lot of verbose versions instead of even being able to use the simple call.

e.g. turning

    public static Task VerifyCodeFixAsync(string source, DiagnosticResult[] expected, string fixedSource)
    {
        var test = new TTest
        {
            TestCode = source,
            FixedCode = fixedSource,
        };

        test.ExpectedDiagnostics.AddRange(expected);
        return test.RunAsync(CancellationToken.None);
    }

into

    public static Task VerifyCodeFixAsync(string source, DiagnosticResult[] expected, string fixedSource, Action<TTest> configureTest)
    {
        var test = new TTest
        {
            TestCode = source,
            FixedCode = fixedSource,
        };

        test.ExpectedDiagnostics.AddRange(expected);
        configureTest?.Invoke(test);
        return test.RunAsync(CancellationToken.None);
    }

That way inheriting from the test would not be necessary anymore and changing the level of compiler diagnostics would be simple + setting the behavior would also involve less code.

taori · 2020-01-19T01:59:51Z

For example this:

var expected = new[]
{
	// Test0.cs(24,21): info ACA0001: Forward execution of "Method1" to member "_disposables"
	Verifier.Diagnostic().WithSpan(24, 21, 24, 28).WithArguments("Method1", "_disposables"),
	// Test0.cs(33,21): info ACA0001: Forward execution of "Method2" to member "_disposables"
	Verifier.Diagnostic().WithSpan(33, 21, 33, 28).WithArguments("Method2", "_disposables"),
	// Test0.cs(38,21): info ACA0001: Forward execution of "Method2" to member "_disposables"
	Verifier.Diagnostic().WithSpan(38, 21, 38, 28).WithArguments("Method2", "_disposables"),
	// Test0.cs(43,21): info ACA0001: Forward execution of "Method3" to member "_disposables"
	Verifier.Diagnostic().WithSpan(43, 21, 43, 28).WithArguments("Method3", "_disposables")
};
await Verifier.VerifyCodeFixAsync(test, expected, fixtest);

right now gives me this:

This could be fixed by simply turning the code to

var expected = new[]
{
	// Test0.cs(24,21): info ACA0001: Forward execution of "Method1" to member "_disposables"
	Verifier.Diagnostic().WithSpan(24, 21, 24, 28).WithArguments("Method1", "_disposables"),
        // Test0.cs(33,21): info ACA0001: Forward execution of "Method2" to member "_disposables"
	Verifier.Diagnostic().WithSpan(33, 21, 33, 28).WithArguments("Method2", "_disposables"),
        // Test0.cs(38,21): info ACA0001: Forward execution of "Method2" to member "_disposables"
	Verifier.Diagnostic().WithSpan(38, 21, 38, 28).WithArguments("Method2", "_disposables"),
        // Test0.cs(43,21): info ACA0001: Forward execution of "Method3" to member "_disposables"
	Verifier.Diagnostic().WithSpan(43, 21, 43, 28).WithArguments("Method3", "_disposables")
};
await Verifier.VerifyCodeFixAsync(test, expected, fixtest, config => config.TestBehaviors = TestBehaviors.SkipGeneratedCodeCheck);

or ofc:
await Verifier.VerifyCodeFixAsync(test, expected, fixtest, SomeConfiguration);

instead of

await new CodeFixTest<Analyzer, FixByForwardingToCollectionChildren>()
{
	TestBehaviors = TestBehaviors.SkipGeneratedCodeCheck,
	CompilerDiagnostics = CompilerDiagnostics.Errors,
	TestState =
	{
		Sources = {test},
		ExpectedDiagnostics =
                {	
			// Test0.cs(24,21): info ACA0001: Forward execution of "Method1" to member "_disposables"	
			Verifier.Diagnostic().WithSpan(24, 21, 24, 28).WithArguments("Method1", "_disposables"),
			// Test0.cs(33,21): info ACA0001: Forward execution of "Method2" to member "_disposables"
                    	Verifier.Diagnostic().WithSpan(33, 21, 33, 28).WithArguments("Method2", "_disposables"),
			// Test0.cs(38,21): info ACA0001: Forward execution of "Method2" to member "_disposables"
                    	Verifier.Diagnostic().WithSpan(38, 21, 38, 28).WithArguments("Method2", "_disposables"),
			// Test0.cs(43,21): info ACA0001: Forward execution of "Method3" to member "_disposables"
			Verifier.Diagnostic().WithSpan(43, 21, 43, 28).WithArguments("Method3", "_disposables")
                    },
		},
		FixedState =
		{
			ExpectedDiagnostics =
			{
                	},
			Sources = {fixtest}
		},
	}.RunAsync();

I know the Configure call, can allow me to fix my test by altering the mode, but from looking at msdocs articles regarding the enum values i can't quite tell whether it is actually okay to change it to None.

sharwell · 2020-01-20T21:49:05Z

what are your thoughts on introducing a settings property to alter the values of the test prior to running it

I'm not a big fan of this right now, for two reasons:

Currently the "full" test form allows for basically anything you need to do, which makes it easy to just have one consistent way to get things done.
We could add a refactoring which converts a test from short form to full form, so if/when you need to do something special in a test, it's not tedious.

That way inheriting from the test would not be necessary anymore

I don't see this as a particular advantage. I find the inherited types quite convenient for tailoring the test framework to the majority characteristics of an analyzer library.

sharwell · 2020-01-20T21:50:53Z

This could be fixed by simply turning the code to ...

There is no need to set properties to their default values. For your example:

CompilerDiagnostics can be omitted to use the default
FixedState.ExpectedDiagnostics can be omitted to use the default

taori · 2020-01-20T21:57:25Z

what are your thoughts on introducing a settings property to alter the values of the test prior to running it

I'm not a big fan of this right now, for two reasons:
1. Currently the "full" test form allows for basically anything you need to do, which makes it easy to just have one consistent way to get things done.

2. We could add a refactoring which converts a test from short form to full form, so if/when you need to do something special in a test, it's not tedious.
That way inheriting from the test would not be necessary anymore

I don't see this as a particular advantage. I find the inherited types quite convenient for tailoring the test framework to the majority characteristics of an analyzer library.

option 2 seems like a viable alternative compared to a configuration delegate, yes.

Yes, it is good that this is possible, but it feels awkward to use in simple use cases because of the required verbosity. I agree that this option should always exist, because it allows highly granular control over how the test will be evaluated, but having the opportunity of slim test cases with low boilerplate should also be an option.

sharwell · 2020-01-20T22:00:13Z

but having the opportunity of slim test cases with low boilerplate should also be an option.

The problem here is each analyzer library tends to have different "minimum functionality requirements". Accounting for each variation of "minimum" results in a library that isn't simple or minimum for anyone. The main advantage of starting with an inherited type in your project is you can define the minimum API as it applies to the patterns/needs of the analyzer project in question, without concerning yourself with the details of any other analyzer.

taori · 2020-01-20T22:06:43Z

Makes sense. In that case perhaps there just have to be a couple more steps in the verifier which need to be extendable to get the desired testing behavior within the scope of an analyzer repo, where the written analyzers are not concerned with codegeneration or the consumer wants to alter the infered expected result generation, you could just do that then.

sharwell · 2020-08-07T15:32:04Z

This was fixed by #569. If the analyzer calls ConfigureGeneratedCodeAnalysis, then the expected test result will adhere to that configuration without needing to specify TestBehaviors.SkipGeneratedCodeCheck.

sharwell added Area-MS.CA.Testing Microsoft.CodeAnalysis.Testing question labels Jan 20, 2020

sharwell self-assigned this Jan 20, 2020

sharwell mentioned this issue Jun 25, 2020

AnalyzerTest.IsSubjectToExclusion constrains DiagnosticsID #535

Closed

sharwell closed this as completed Aug 7, 2020

sharwell mentioned this issue Aug 7, 2020

Automatically detect generated code configuration #569

Merged

sharwell added the fixed label Aug 7, 2020

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Special test scenario with new unit test framework fails #422

Special test scenario with new unit test framework fails #422

taori commented Jan 17, 2020

sharwell commented Jan 17, 2020

taori commented Jan 17, 2020

sharwell commented Jan 17, 2020

taori commented Jan 17, 2020

sharwell commented Jan 17, 2020 •

edited

Loading

taori commented Jan 18, 2020

sharwell commented Jan 18, 2020

taori commented Jan 18, 2020

taori commented Jan 18, 2020

taori commented Jan 19, 2020 •

edited

Loading

sharwell commented Jan 20, 2020

sharwell commented Jan 20, 2020

taori commented Jan 20, 2020

sharwell commented Jan 20, 2020

taori commented Jan 20, 2020

sharwell commented Aug 7, 2020

Special test scenario with new unit test framework fails #422

Special test scenario with new unit test framework fails #422

Comments

taori commented Jan 17, 2020

sharwell commented Jan 17, 2020

taori commented Jan 17, 2020

sharwell commented Jan 17, 2020

taori commented Jan 17, 2020

sharwell commented Jan 17, 2020 • edited Loading

taori commented Jan 18, 2020

sharwell commented Jan 18, 2020

taori commented Jan 18, 2020

taori commented Jan 18, 2020

taori commented Jan 19, 2020 • edited Loading

sharwell commented Jan 20, 2020

sharwell commented Jan 20, 2020

taori commented Jan 20, 2020

sharwell commented Jan 20, 2020

taori commented Jan 20, 2020

sharwell commented Aug 7, 2020

sharwell commented Jan 17, 2020 •

edited

Loading

taori commented Jan 19, 2020 •

edited

Loading