RunTests.tt

<#@ template debug="false" hostspecific="True" language="C#" #>
<#@ assembly name="System.Core" #>
<#@ import namespace="System.Linq" #>
<#@ import namespace="System.IO" #>
<#@ import namespace="System.Globalization" #>
<#@ import namespace="System.Text" #>
<#@ import namespace="System.Text.RegularExpressions" #>
<#@ import namespace="System.Collections.Generic" #>
<#@ assembly name="%NewtonsoftDependencyValue%" #>
<#@ import namespace="Newtonsoft.Json" #>
<#@ output extension=".cs" #>

<#

var testRoot = Path.Combine(Path.GetDirectoryName(Host.TemplateFile), @"..\..\test");

List<Test> tests;
using (StreamReader sr = File.OpenText(Path.Combine(testRoot, "core.vwtest.json")))
{
    tests = JsonConvert.DeserializeObject<List<Test>>(sr.ReadToEnd());
}

var skipList = new[] { 13, 32, 39, 258, 40, 259, 41, 260, 59, 60, 61, 66, 68, 90,
    25, 26, 349, 350, 356, 357, 358, // crash
    92, 95, 96, 98,	91, 99, 118, 119, 120,
    176, 177, 207, 208, //depend on shell scripts for input/output
    14, 16, 17, 31, 33, 34,53, 101, 102, 103, 105, 106, 111, 112, 412, 413, 414, 422, 423, 424, // float delta
    71, // --examples to test parser
    143, 144, 146, 158, 189, 202, 237, 312, 316, 318, 319, 324, 325, 326, 347, 351, 348, // native json parsing
    149, 152, 156, 193, 194, 217, 385, // bash script
    188, // possibly float delta
    195, 275, 276, //--onethread is a shell option, not available via library
    203, // Cluster mode test
    204, // Testing option usage
    216, // DSJSON not supported
    220, // deamon mode shell script test not supported
    227, // "single continuous action, pdf value" prediction type for cats not yet supported
    229,  // "continous action with probability density function" prediction type for cats_pdf not yet supported
    239, 240, 241, 242, 243, 244, 245, 246, // flatbuffer input tests, to be enabled in due course
    256, 299, 300, 306, 310, 311, 327, 328, 329, 330, 331, 367, 368, 396, 397, 398, // DSJSON not supported
    383, 389, 390, 391, 392, 393, // no data file
    400, 404, // positional args
    405, 406, 407, 411, 415, 417, 456, 457 // DSJSON not supported
    };

var outputModels = new Dictionary<string, TestCase>();
var testcases = new Dictionary<int, TestCase>();

for (var i = 0; i < tests.Count; i++)
{
    if (tests[i].vw_command == null ||  tests[i].desc.Contains("SkipC#"))
    {
        continue;
    }

    TestCase testcase = new TestCase();

    testcase.Id = tests[i].id;
    testcase.Comment = tests[i].desc.Replace("\"", "\"\"");
    testcase.Arguments = tests[i].vw_command;
    testcase.InputData = MatchArgument(tests[i].vw_command, "-d");
    testcase.InitialRegressor = MatchArgument(tests[i].vw_command, "-i");
    testcase.FinalRegressor = MatchArgument(tests[i].vw_command, "-f");

    foreach(KeyValuePair<string, string> kvPair in tests[i].diff_files)
    {
        if (kvPair.Key == "stderr")
        {
            testcase.Stderr= kvPair.Value;
        }
        else if (kvPair.Key.EndsWith(".predict"))
        {
            testcase.Predict = kvPair.Value;
        }
    }

    // resolve dependencies
     if (!string.IsNullOrEmpty(testcase.FinalRegressor))
            outputModels[testcase.FinalRegressor] = testcase;

    if (!string.IsNullOrEmpty(testcase.InitialRegressor))
    {
        TestCase dep;
        bool foundInitialRegressor = false;
        if (outputModels.TryGetValue(testcase.InitialRegressor, out dep))
        {
            testcase.Dependency = dep;
            foundInitialRegressor = true;
        }
        else if (testcase.InitialRegressor.StartsWith("model-sets/"))
        {
            string fullPath = Path.Combine(testRoot, testcase.InitialRegressor);
            foundInitialRegressor = File.Exists(fullPath);
        }

        if (!foundInitialRegressor)
        {
            throw new Exception("Missing dependency: '" + testcase.InitialRegressor + "' for test case " + testcase.Id);
        }
    }


    if (testcases.ContainsKey(testcase.Id))
        throw new Exception("Testcase IDs must be unique: duplicate # Test " + testcase.Id + " in line " + i + " total tests cases: " + testcases.Count);


    testcases.Add(testcase.Id, testcase);
}
#>

using Microsoft.VisualStudio.TestTools.UnitTesting;
using System.IO;
using System.IO.Compression;
using VW;

namespace cs_unittest
{
    [TestClass]
    public partial class RunTests : TestBase
    {
<# foreach (var mainTestcase in testcases.Values) {
        if (mainTestcase.Id == 0)
            continue;
#>
        [TestMethod]
        [Description(@"<#=(mainTestcase.Comment ?? "").Trim()#>")]

<# if (skipList.Contains(mainTestcase.Id)) { #>
        [Ignore]
<# } #>
        [TestCategory("Vowpal Wabbit/Command Line")]
        public void CommandLine_Test<#=mainTestcase.Id#>()
        {
<# foreach (var tc in mainTestcase.InDependencyOrder()) { #>
            RunTestsHelper.ExecuteTest(
                <#=tc.Id#>,
                "<#=tc.Arguments#>",
                "<#=tc.InputData#>",
                "<#=tc.Stderr#>",
                "<#=tc.Predict#>");
<# } #>
        }

<# } #>
    }
}

<#+
// Used to parse the json file
public class Test
{
    public int id { get; set; }
    public string desc { get; set; }
    public string vw_command { get; set; }
    public string bash_command { get; set; }
    public Dictionary<string, string> diff_files { get; set; }
    public IList<string> input_files { get; set; }
    public IList<int> depends_on { get; set; }
}

class TestCase
{
    public int Id;

    public string Arguments = "";

    public string InitialRegressor;

    public string FinalRegressor;

    public string InputData = "";

    public string Stderr = "";

    public string Predict = "";

    public string Comment;

    public TestCase Dependency;

    public List<TestCase> InDependencyOrder()
    {
        var tests = new List<TestCase>();

        var dep = this;
        while (dep != null)
        {
            tests.Add(dep);
            dep = dep.Dependency;
        }

        tests.Reverse();
        return tests;
    }
}

private string MatchArgument(string args, string option)
{
    var match = Regex.Match(args, Regex.Escape(option) + @"\s+(?<value>\S+)");
    return match.Success ? match.Groups["value"].Value : "";
}

#>