machinelearning/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs at 17ee205e585beb62777475af6d59cba816675eeb · dotnet/machinelearning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using BenchmarkDotNet.Attributes;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.FastTree;
using Microsoft.ML.Runtime.LightGBM;
using Microsoft.ML.Runtime.RunTests;
using Microsoft.ML.Runtime.Tools;
using Microsoft.ML.Transforms;
using System.IO;

namespace Microsoft.ML.Benchmarks
{
    [Config(typeof(TrainConfig))]
    public class RankingTrain
    {
        private string _mslrWeb10k_Validate;
        private string _mslrWeb10k_Train;

        [GlobalSetup]
        public void SetupTrainingSpeedTests()
        {
            _mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
            _mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);

            if (!File.Exists(_mslrWeb10k_Validate))
                throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate));

            if (!File.Exists(_mslrWeb10k_Train))
                throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Train));
        }

        [Benchmark]
        public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
        {
            string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
                " eval=RankingEvaluator{t=10}" +
                " data=" + _mslrWeb10k_Train +
                " loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
                " xf=HashTransform{col=GroupId} xf=NAHandleTransform{col=Features}" +
                " tr=FastTreeRanking{}";

            using (var environment = EnvironmentFactory.CreateRankingEnvironment<RankerEvaluator, TextLoader, HashTransformer, FastTreeRankingTrainer>())
            {
                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
            }
        }

        [Benchmark]
        public void TrainTest_Ranking_MSLRWeb10K_RawNumericFeatures_LightGBMRanking()
        {
            string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
                " eval=RankingEvaluator{t=10}" +
                " data=" + _mslrWeb10k_Train +
                " loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
                " xf=HashTransform{col=GroupId}" +
                " xf=NAHandleTransform{col=Features}" +
                " tr=LightGBMRanking{}";

            using (var environment = EnvironmentFactory.CreateRankingEnvironment<RankerEvaluator, TextLoader, HashTransformer, LightGbmMulticlassTrainer>())
            {
                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
            }
        }
    }

    public class RankingTest
    {
        private string _mslrWeb10k_Validate;
        private string _mslrWeb10k_Train;
        private string _mslrWeb10k_Test;
        private string _modelPath_MSLR;

        [GlobalSetup]
        public void SetupScoringSpeedTests()
        {
            _mslrWeb10k_Test = Path.GetFullPath(TestDatasets.MSLRWeb.testFilename);
            _mslrWeb10k_Validate = Path.GetFullPath(TestDatasets.MSLRWeb.validFilename);
            _mslrWeb10k_Train = Path.GetFullPath(TestDatasets.MSLRWeb.trainFilename);

            if (!File.Exists(_mslrWeb10k_Test))
                throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Test));

            if (!File.Exists(_mslrWeb10k_Validate))
                throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Validate));

            if (!File.Exists(_mslrWeb10k_Train))
                throw new FileNotFoundException(string.Format(Errors.DatasetNotFound, _mslrWeb10k_Train));

            _modelPath_MSLR = Path.Combine(Directory.GetCurrentDirectory(), @"FastTreeRankingModel.zip");

            string cmd = @"TrainTest test=" + _mslrWeb10k_Validate +
                " eval=RankingEvaluator{t=10}" +
                " data=" + _mslrWeb10k_Train +
                " loader=TextLoader{col=Label:R4:0 col=GroupId:TX:1 col=Features:R4:2-138}" +
                " xf=HashTransform{col=GroupId}" +
                " xf=NAHandleTransform{col=Features}" +
                " tr=FastTreeRanking{}" +
                " out={" + _modelPath_MSLR + "}";

            using (var environment = EnvironmentFactory.CreateRankingEnvironment<RankerEvaluator, TextLoader, HashTransformer, FastTreeRankingTrainer>())
            {
                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
            }
        }

        [Benchmark]
        public void Test_Ranking_MSLRWeb10K_RawNumericFeatures_FastTreeRanking()
        {
            // This benchmark is profiling bulk scoring speed and not training speed.
            string cmd = @"Test data=" + _mslrWeb10k_Test + " in=" + _modelPath_MSLR;

            using (var environment = EnvironmentFactory.CreateRankingEnvironment<RankerEvaluator, TextLoader, HashTransformer, FastTreeRankingTrainer>())
            {
                Maml.MainCore(environment, cmd, alwaysPrintStacktrace: false);
            }
        }
    }
}