Skip to content

Commit f903157

Browse files
authored
Merge pull request #23 from problem-judge/dev/metrics
Add metrics
2 parents 639b2c4 + 74cb9fb commit f903157

File tree

28 files changed

+555
-351
lines changed

28 files changed

+555
-351
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
testing_system
33
cp.sh
44
swag
5+
custom

common/connectors/masterconn/structs.go

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,22 @@ import (
44
"testing_system/common/connectors/invokerconn"
55
"testing_system/common/constants/verdict"
66
"testing_system/lib/customfields"
7+
"time"
78
)
89

910
type InvokerJobResult struct {
10-
JobID string `json:"JobID"`
11+
Job *invokerconn.Job `json:"Job"`
1112

1213
Verdict verdict.Verdict `json:"Verdict"`
1314
Points *float64 `json:"Points,omitempty"`
1415

1516
Error string `json:"Error,omitempty"` // Is set only in case of check failed caused by invoker problems
1617

17-
Statistics *JobResultStatistics `json:"Statistics"`
18+
Statistics *JobResultStatistics `json:"Statistics,omitempty"`
1819

1920
InvokerStatus *invokerconn.Status `json:"InvokerStatus"`
21+
22+
Metrics *InvokerJobMetrics `json:"Metrics"`
2023
}
2124

2225
type JobResultStatistics struct {
@@ -27,3 +30,13 @@ type JobResultStatistics struct {
2730
ExitCode int `json:"ExitCode"`
2831
// TODO: Add more statistics
2932
}
33+
34+
type InvokerJobMetrics struct {
35+
TestingWaitDuration time.Duration `json:"InvokerWaitDuration"`
36+
TotalSandboxOccupation time.Duration `json:"TotalSandboxOccupation"`
37+
ResourceWaitDuration time.Duration `json:"ResourceWaitDuration"`
38+
FileActionsDuration time.Duration `json:"FileActionsDuration"`
39+
ExecutionWaitDuration time.Duration `json:"ExecutionWaitDuration"`
40+
ExecutionDuration time.Duration `json:"ExecutionDuration"`
41+
SendResultDuration time.Duration `json:"SendResultDuration"`
42+
}

common/metrics/metrics.go

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package metrics
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"testing_system/common/connectors/masterconn"
6+
)
7+
8+
const (
9+
invokerLabel = "invoker"
10+
jobTypeLabel = "job_type"
11+
)
12+
13+
type Collector struct {
14+
Registerer *prometheus.Registry
15+
16+
InvokerJobResults *prometheus.CounterVec
17+
InvokerTestingWaitDuration *prometheus.CounterVec
18+
InvokerSandboxOccupationDuration *prometheus.CounterVec
19+
InvokerResourceWaitDuration *prometheus.CounterVec
20+
InvokerFileActionsDuration *prometheus.CounterVec
21+
InvokerExecutionWaitDuration *prometheus.CounterVec
22+
InvokerExecutionDuration *prometheus.CounterVec
23+
InvokerSendResultDuration *prometheus.CounterVec
24+
}
25+
26+
func NewCollector() *Collector {
27+
c := &Collector{
28+
Registerer: prometheus.NewRegistry(),
29+
}
30+
c.InvokerJobResults = c.createInvokerCounter(
31+
"job_results_count",
32+
"Number of job results received from invoker",
33+
)
34+
35+
c.InvokerTestingWaitDuration = c.createInvokerCounter(
36+
"testing_wait_duration_sum",
37+
"Time submission waits for testing in invoker",
38+
)
39+
40+
c.InvokerSandboxOccupationDuration = c.createInvokerCounter(
41+
"sandbox_occupation_duration_sum",
42+
"Total sandbox time for submission testing in invoker",
43+
)
44+
45+
c.InvokerResourceWaitDuration = c.createInvokerCounter(
46+
"resource_wait_duration_sum",
47+
"Total time spent waiting for resources for submissions to load in invokers",
48+
)
49+
50+
c.InvokerFileActionsDuration = c.createInvokerCounter(
51+
"file_actions_duration_sum",
52+
"Total time spent waiting for file copy to sandbox in invoker",
53+
)
54+
55+
c.InvokerExecutionWaitDuration = c.createInvokerCounter(
56+
"execution_wait_duration_sum",
57+
"Total time spent waiting for execution of process on invoker when sandbox is set up",
58+
)
59+
60+
c.InvokerExecutionDuration = c.createInvokerCounter(
61+
"execution_duration_sum",
62+
"Total time spent on executing processes in sandboxes",
63+
)
64+
65+
c.InvokerSendResultDuration = c.createInvokerCounter(
66+
"send_result_duration_sum",
67+
"Total time spent on sending results from invoker to storage",
68+
)
69+
return c
70+
}
71+
72+
func (c *Collector) createInvokerCounter(
73+
name string,
74+
help string,
75+
) *prometheus.CounterVec {
76+
counter := prometheus.NewCounterVec(
77+
prometheus.CounterOpts{
78+
Namespace: "ts",
79+
Subsystem: "invoker",
80+
Name: name,
81+
Help: help,
82+
},
83+
[]string{invokerLabel, jobTypeLabel},
84+
)
85+
c.Registerer.MustRegister(counter)
86+
return counter
87+
}
88+
89+
func (c *Collector) NewJobResult(result *masterconn.InvokerJobResult) {
90+
labels := prometheus.Labels{
91+
invokerLabel: result.InvokerStatus.Address,
92+
jobTypeLabel: result.Job.Type.String(),
93+
}
94+
95+
c.InvokerJobResults.With(labels).Inc()
96+
c.InvokerTestingWaitDuration.With(labels).Add(result.Metrics.TestingWaitDuration.Seconds())
97+
c.InvokerSandboxOccupationDuration.With(labels).Add(result.Metrics.TotalSandboxOccupation.Seconds())
98+
c.InvokerResourceWaitDuration.With(labels).Add(result.Metrics.ResourceWaitDuration.Seconds())
99+
c.InvokerFileActionsDuration.With(labels).Add(result.Metrics.FileActionsDuration.Seconds())
100+
c.InvokerExecutionWaitDuration.With(labels).Add(result.Metrics.ExecutionWaitDuration.Seconds())
101+
c.InvokerExecutionDuration.With(labels).Add(result.Metrics.ExecutionDuration.Seconds())
102+
c.InvokerSendResultDuration.With(labels).Add(result.Metrics.SendResultDuration.Seconds())
103+
}

common/testing_system.go

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package common
33
import (
44
"context"
55
"fmt"
6+
"github.com/prometheus/client_golang/prometheus/promhttp"
67
"os/signal"
78
"runtime"
89
"slices"
@@ -12,16 +13,18 @@ import (
1213
"testing_system/common/connectors/masterconn"
1314
"testing_system/common/connectors/storageconn"
1415
"testing_system/common/db"
16+
"testing_system/common/metrics"
1517
"testing_system/lib/logger"
1618

1719
"github.com/gin-gonic/gin"
1820
"gorm.io/gorm"
1921
)
2022

2123
type TestingSystem struct {
22-
Config *config.Config
23-
Router *gin.Engine
24-
DB *gorm.DB
24+
Config *config.Config
25+
Router *gin.Engine
26+
DB *gorm.DB
27+
Metrics *metrics.Collector
2528

2629
MasterConn *masterconn.Connector
2730
StorageConn *storageconn.Connector
@@ -54,6 +57,11 @@ func InitTestingSystem(configPath string) *TestingSystem {
5457
ts.MasterConn = masterconn.NewConnector(ts.Config.MasterConnection)
5558
ts.StorageConn = storageconn.NewConnector(ts.Config.StorageConnection)
5659

60+
ts.Metrics = metrics.NewCollector()
61+
ts.Router.GET("/metrics", gin.WrapH(promhttp.HandlerFor(ts.Metrics.Registerer, promhttp.HandlerOpts{
62+
ErrorLog: logger.CreateWriter(logger.LogLevelError, "[Prometheus]"),
63+
})))
64+
5765
return ts
5866
}
5967

data/compile/config.yaml

Lines changed: 0 additions & 8 deletions
This file was deleted.

data/compile/scripts/g++.sh.tmpl

Lines changed: 0 additions & 3 deletions
This file was deleted.

go.mod

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/gin-gonic/gin v1.10.0
88
github.com/go-resty/resty/v2 v2.16.5
99
github.com/google/uuid v1.6.0
10+
github.com/prometheus/client_golang v1.22.0
1011
github.com/stretchr/testify v1.10.0
1112
github.com/swaggo/files v1.0.1
1213
github.com/swaggo/gin-swagger v1.6.0
@@ -20,8 +21,10 @@ require (
2021

2122
require (
2223
github.com/KyleBanks/depth v1.2.1 // indirect
24+
github.com/beorn7/perks v1.0.1 // indirect
2325
github.com/bytedance/sonic v1.13.2 // indirect
2426
github.com/bytedance/sonic/loader v0.2.4 // indirect
27+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
2528
github.com/cloudwego/base64x v0.1.5 // indirect
2629
github.com/davecgh/go-spew v1.1.1 // indirect
2730
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
@@ -49,8 +52,12 @@ require (
4952
github.com/mattn/go-sqlite3 v1.14.22 // indirect
5053
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
5154
github.com/modern-go/reflect2 v1.0.2 // indirect
55+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
5256
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
5357
github.com/pmezard/go-difflib v1.0.0 // indirect
58+
github.com/prometheus/client_model v0.6.1 // indirect
59+
github.com/prometheus/common v0.62.0 // indirect
60+
github.com/prometheus/procfs v0.15.1 // indirect
5461
github.com/rogpeppe/go-internal v1.13.1 // indirect
5562
github.com/swaggo/swag v1.16.4 // indirect
5663
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect

go.sum

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
22
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
3+
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
4+
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
35
github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
46
github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
57
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
68
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
79
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
810
github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8=
911
github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
12+
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
13+
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
1014
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
1115
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
1216
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
@@ -41,8 +45,8 @@ github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptd
4145
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
4246
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
4347
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
44-
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
45-
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
48+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
49+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
4650
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
4751
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
4852
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -83,10 +87,20 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
8387
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
8488
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
8589
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
90+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
91+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
8692
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
8793
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
8894
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
8995
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
96+
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
97+
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
98+
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
99+
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
100+
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
101+
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
102+
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
103+
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
90104
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
91105
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
92106
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

invoker/check_pipeline.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func (s *JobPipelineState) generateCheckerRunConfig() error {
5858

5959
func (s *JobPipelineState) executeCheckerRunCommand() error {
6060
s.executeWaitGroup.Add(1)
61-
s.invoker.RunQueue <- s.runChecker
61+
s.runProcess(s.runChecker)
6262
s.executeWaitGroup.Wait()
6363

6464
if s.test.checkResult.Err != nil {
@@ -175,11 +175,11 @@ func (s *JobPipelineState) uploadTestRunResources() error {
175175
func (s *JobPipelineState) uploadCheckerOutput() error {
176176
checkerOutputRequest := &storageconn.Request{
177177
Resource: resource.CheckerOutput,
178-
SubmitID: uint64(s.job.Submission.ID),
178+
SubmitID: uint64(s.job.submission.ID),
179179
TestID: s.job.Test,
180180
File: s.test.checkerOutputReader,
181181
}
182-
resp := s.invoker.TS.StorageConn.Upload(checkerOutputRequest)
182+
resp := s.uploadResource(checkerOutputRequest)
183183
if resp.Error != nil {
184184
return fmt.Errorf("can not upload checker output to storage, error: %s", resp.Error.Error())
185185
}

0 commit comments

Comments
 (0)