Introduce Test System based on Traits

Silk.NET.sln

@@ -80,6 +80,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Silk.NET.SilkTouch.Scraper.
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Silk.NET.SilkTouch.IntegrationTests", "tests\Silk.NET.SilkTouch.IntegrationTests\Silk.NET.SilkTouch.IntegrationTests.csproj", "{66FE736C-C407-44C3-A94E-4345E22AA95E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Silk.NET.SilkTouch.TestFramework", "tests\Silk.NET.SilkTouch.TestFramework\Silk.NET.SilkTouch.TestFramework.csproj", "{381D1039-3259-488F-BB25-D90EE63A3E82}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -330,6 +332,18 @@ Global
 		{66FE736C-C407-44C3-A94E-4345E22AA95E}.Release|x64.Build.0 = Release|Any CPU
 		{66FE736C-C407-44C3-A94E-4345E22AA95E}.Release|x86.ActiveCfg = Release|Any CPU
 		{66FE736C-C407-44C3-A94E-4345E22AA95E}.Release|x86.Build.0 = Release|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Debug|x64.Build.0 = Debug|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Debug|x86.Build.0 = Debug|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Release|Any CPU.Build.0 = Release|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Release|x64.ActiveCfg = Release|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Release|x64.Build.0 = Release|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Release|x86.ActiveCfg = Release|Any CPU
+		{381D1039-3259-488F-BB25-D90EE63A3E82}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -361,6 +375,7 @@ Global
 		{795A93A6-9578-439F-BB08-07B148B1D4CE} = {94D5D1E1-B998-4CB1-9D04-DA138A2B0F3C}
 		{5329AC43-7177-4953-AFAB-A9FA7B9A4C7C} = {94D5D1E1-B998-4CB1-9D04-DA138A2B0F3C}
 		{66FE736C-C407-44C3-A94E-4345E22AA95E} = {94D5D1E1-B998-4CB1-9D04-DA138A2B0F3C}
+		{381D1039-3259-488F-BB25-D90EE63A3E82} = {94D5D1E1-B998-4CB1-9D04-DA138A2B0F3C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F5273D7F-3334-48DF-94E3-41AE6816CD4D}

documentation/for-contributors/generators/testing.md

@@ -0,0 +1,30 @@
+# Testing
+
+SilkTouch Generators should be extensively Tested.
+
+## Unit Tests
+
+### Traits
+
+All Unit & Integration tests should be flagged with the appropriate Category (Emitter, Scraper, Symbols, Integration) using `Trait("Category", "MyCategory")`.
+Additionally the following Traits should be applied where appropriate:
+
+-   `Target Language` this should be applied wherever a specific target language is tested (valid Values: `C#`)
+-   `Source Language` this should be applied wherever a specific source language is tested (valid Values: `C++`)
+-   `Feature` this should be applied wherever a specific feature is covered, ie `Structs`, `Fields`, `Namespaces`, etc. These at the same time are used as Feature Flags (toggled in Silk.NET.SilkTouch.TestFramework). Tests with features that are not yet implemeneted may be created and simply turned off via these feature flags. (valid Values: not tracked. check `tests/Silk.NET.SilkTouch.TestFramework/SilkTouchTestFramework.cs`)
+
+### Symbol Layer
+
+Unit Tests need to be written for all symbols AND the Symbol Visitor. They should cover that the symbol visitor visits these symbols and their children correctly and any functionality the symbols themselves might implement
+
+### Scraper
+
+Tests should be written covering XML -> Symbol conversion. No C(++) -> XML tests are needed as this is the job of clang (sharp).
+
+### Emitter
+
+Tests should be written covering Symbol -> C# conversion.
+
+### Integration
+
+Integration Tests should exist to cover most if not all areas of the generator. These are simply a sanity check & samples of what the generator can do. Unit Tests are still required.

tests/Silk.NET.SilkTouch.Emitter.Tests/Assembly.cs

@@ -0,0 +1,6 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit;
+
+[assembly: TestFramework("Silk.NET.SilkTouch.TestFramework.SilkTouchTestFramework", "Silk.NET.SilkTouch.TestFramework")]

tests/Silk.NET.SilkTouch.Emitter.Tests/EmitterFieldTests.cs

@@ -13,7 +13,10 @@ namespace Silk.NET.SilkTouch.Emitter.Tests;
 
 public sealed class EmitterFieldIntegrationTests : EmitterTest
 {
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Fields"),
+    Trait("Target Language", "C#")]
     public void FieldIntegration()
     {
         var syntax = Transform(new FieldSymbol(new StructSymbol(new IdentifierSymbol("int"),ImmutableArray<FieldSymbol>.Empty), new IdentifierSymbol("Test")));
@@ -22,7 +25,10 @@ public void FieldIntegration()
         Assert.Equal("public int Test;", result);
     }
 
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Fields"),
+     Trait("Target Language", "C#")]
     public void FieldIsPublic()
     {
         var syntax = Transform
@@ -38,7 +44,10 @@ public void FieldIsPublic()
         Assert.Single(syntax!.Modifiers, x => x.IsKind(SyntaxKind.PublicKeyword));
     }
 
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Fields"),
+     Trait("Target Language", "C#")]
     public void CorrectTypeIdentifier()
     {
         var syntax = Transform
@@ -56,7 +65,10 @@ public void CorrectTypeIdentifier()
         Assert.Equal("int", type!.Identifier.Text);
     }
 
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Fields"),
+     Trait("Target Language", "C#")]
     public void CorrectIdentifier()
     {
         var syntax = Transform

tests/Silk.NET.SilkTouch.Emitter.Tests/EmitterNamespaceMemberTests.cs

@@ -10,7 +10,10 @@ namespace Silk.NET.SilkTouch.Emitter.Tests;
 
 public class EmitterNamespaceMemberTests : EmitterTest
 {
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Namespaces"),
+     Trait("Target Language", "C#")]
     public void SingleMemberIntegration()
     {
         var syntax = Transform(new NamespaceSymbol(new IdentifierSymbol("Test"), new []
@@ -22,7 +25,10 @@ public void SingleMemberIntegration()
         Assert.Equal("namespace Test\n{\npublic struct Test2\n{\n}\n}\n", result);
     }
 
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Namespaces"),
+     Trait("Target Language", "C#")]
     public void MultipleMembersIntegration()
     {
         var syntax = Transform(new NamespaceSymbol(new IdentifierSymbol("Test"), new []

tests/Silk.NET.SilkTouch.Emitter.Tests/EmitterNamespaceTests.cs

@@ -12,7 +12,10 @@ namespace Silk.NET.SilkTouch.Emitter.Tests;
 
 public class EmitterNamespaceTests : EmitterTest
 {
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Namespaces"),
+     Trait("Target Language", "C#")]
     public void NamespaceIntegration()
     {
         var syntax = Transform(new NamespaceSymbol(new IdentifierSymbol("Test"), ImmutableArray<TypeSymbol>.Empty));
@@ -21,7 +24,10 @@ public void NamespaceIntegration()
         Assert.Equal("namespace Test\n{\n}\n", result);
     }
 
-    [Fact]
+    [Fact, 
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Namespaces"),
+     Trait("Target Language", "C#")]
     public void CorrectIdentifier()
     {
         var syntax = Transform

tests/Silk.NET.SilkTouch.Emitter.Tests/EmitterStructMemberFieldsTests.cs

@@ -9,7 +9,12 @@ namespace Silk.NET.SilkTouch.Emitter.Tests;
 
 public class EmitterStructMemberFieldsTests : EmitterTest
 {
-    [Fact]
+
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Feature", "Fields"),
+     Trait("Target Language", "C#")]
     public void SingleFieldIntegration()
     {
         var node = Transform
@@ -37,7 +42,11 @@ public void SingleFieldIntegration()
         );
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Feature", "Fields"),
+     Trait("Target Language", "C#")]
     public void MultipleFieldsIntegration()
     {
         var node = Transform

tests/Silk.NET.SilkTouch.Emitter.Tests/EmitterStructTests.cs

@@ -10,36 +10,52 @@ namespace Silk.NET.SilkTouch.Emitter.Tests;
 
 public sealed class EmitterStructTests : EmitterTest
 {
-    [Fact]
+
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Target Language", "C#")]
     public void StructSyntax()
     {
         var syntax = Transform(new StructSymbol(new IdentifierSymbol("Test"), ImmutableArray<FieldSymbol>.Empty));
         Assert.IsType<StructDeclarationSyntax>(syntax);
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Target Language", "C#")]
     public void StructKeyword()
     {
         var syntax = Transform(new StructSymbol(new IdentifierSymbol("Test"), ImmutableArray<FieldSymbol>.Empty)) as StructDeclarationSyntax;
         Assert.Equal("struct", syntax!.Keyword.Text);
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Target Language", "C#")]
     public void CorrectIdentifier()
     {
         var syntax = Transform(new StructSymbol(new IdentifierSymbol("Test"), ImmutableArray<FieldSymbol>.Empty)) as StructDeclarationSyntax;
         Assert.Equal("Test", syntax!.Identifier.Text);
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Target Language", "C#")]
     public void IsOnlyPublic()
     {
         var syntax = Transform(new StructSymbol(new IdentifierSymbol("Test"), ImmutableArray<FieldSymbol>.Empty)) as StructDeclarationSyntax;
         var @public = Assert.Single(syntax!.Modifiers);
         Assert.Equal("public", @public.Text);
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Feature", "Structs"),
+     Trait("Target Language", "C#")]
     public void IntegrationEmptyStruct()
     {
         // Note that this test also covers trivia, which is not checked otherwise.

tests/Silk.NET.SilkTouch.Emitter.Tests/IdentifierTests.cs

@@ -8,7 +8,9 @@ namespace Silk.NET.SilkTouch.Emitter.Tests;
 
 public sealed class IdentifierTests : EmitterTest
 {
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Target Language", "C#")]
     public void IdentifierHasNoLeadingTrivia()
     {
         var node = Transform(new IdentifierSymbol("Test"));
@@ -17,7 +19,9 @@ public void IdentifierHasNoLeadingTrivia()
         Assert.False(node.HasLeadingTrivia);
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Target Language", "C#")]
     public void IdentifierHasNoTrailingTrivia()
     {
         var node = Transform(new IdentifierSymbol("Test"));
@@ -26,7 +30,9 @@ public void IdentifierHasNoTrailingTrivia()
         Assert.False(node.HasTrailingTrivia);
     }
 
-    [Fact]
+    [Fact,
+     Trait("Category", "Emitter"),
+     Trait("Target Language", "C#")]
     public void IdentifierIntegration()
     {
         var node = Transform(new IdentifierSymbol("Test"));

tests/Silk.NET.SilkTouch.Emitter.Tests/Silk.NET.SilkTouch.Emitter.Tests.csproj

@@ -18,6 +18,7 @@
 
     <ItemGroup>
       <ProjectReference Include="..\..\src\generators\Silk.NET.SilkTouch.Emitter\Silk.NET.SilkTouch.Emitter.csproj" />
+      <ProjectReference Include="..\Silk.NET.SilkTouch.TestFramework\Silk.NET.SilkTouch.TestFramework.csproj" />
     </ItemGroup>
 
 </Project>

tests/Silk.NET.SilkTouch.IntegrationTests/Assembly.cs

@@ -0,0 +1,6 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Xunit;
+
+[assembly: TestFramework("Silk.NET.SilkTouch.TestFramework.SilkTouchTestFramework", "Silk.NET.SilkTouch.TestFramework")]

tests/Silk.NET.SilkTouch.IntegrationTests/Silk.NET.SilkTouch.IntegrationTests.csproj

@@ -26,6 +26,7 @@
         <ProjectReference Include="..\..\src\generators\Silk.NET.SilkTouch.Scraper\Silk.NET.SilkTouch.Scraper.csproj" />
         <ProjectReference Include="..\..\src\generators\Silk.NET.SilkTouch.Emitter\Silk.NET.SilkTouch.Emitter.csproj" />
         <ProjectReference Include="..\..\src\generators\Silk.NET.SilkTouch.Symbols\Silk.NET.SilkTouch.Symbols.csproj" />
+        <ProjectReference Include="..\Silk.NET.SilkTouch.TestFramework\Silk.NET.SilkTouch.TestFramework.csproj" />
     </ItemGroup>
 
 </Project>