From 3e7b7429c0259fb376b7a5ae054867fa5d9e7daf Mon Sep 17 00:00:00 2001 From: Philip Gribov Date: Tue, 7 Jan 2025 01:33:33 +1100 Subject: [PATCH 1/5] Add common testing system initialization --- go.mod | 32 ++++++++++++- go.sum | 91 ++++++++++++++++++++++++++++++++++-- lib/common/testing_system.go | 38 +++++++++++++++ lib/config/config.go | 47 +++++++++++++++++++ lib/config/connection.go | 10 ++++ lib/logger/logger.go | 79 ++++++++++++++++++++++++++++++- lib/logger/utils.go | 36 ++++++++++++++ main.go | 13 ++++++ 8 files changed, 337 insertions(+), 9 deletions(-) create mode 100644 lib/common/testing_system.go create mode 100644 lib/config/config.go create mode 100644 lib/config/connection.go create mode 100644 lib/logger/utils.go create mode 100644 main.go diff --git a/go.mod b/go.mod index e6decde..945271b 100644 --- a/go.mod +++ b/go.mod @@ -5,19 +5,47 @@ go 1.22 toolchain go1.22rc1 require ( + github.com/gin-gonic/gin v1.10.0 github.com/lib/pq v1.10.9 + github.com/xorcare/pointer v1.2.2 + gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.9 gorm.io/gorm v1.25.12 ) require ( + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - golang.org/x/crypto v0.17.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index 68c9f41..ed323f6 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,34 @@ +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= @@ -13,22 +41,73 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xorcare/pointer v1.2.2 h1:zjD77b5DTehClND4MK+9dDE0DcpFIZisAJ/+yVJvKYA= +github.com/xorcare/pointer v1.2.2/go.mod h1:azsKh7oVwYB7C1o8P284fG8MvtErX/F5/dqXiaj71ak= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -36,3 +115,5 @@ gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/lib/common/testing_system.go b/lib/common/testing_system.go new file mode 100644 index 0000000..ae4c937 --- /dev/null +++ b/lib/common/testing_system.go @@ -0,0 +1,38 @@ +package common + +import ( + "github.com/gin-gonic/gin" + "strconv" + "testing_system/lib/config" + "testing_system/lib/logger" +) + +type TestingSystem struct { + Config *config.Config + Router *gin.Engine + // TODO: Add common params for all testing system components +} + +func InitTestingSystem(configPath string) *TestingSystem { + ts := &TestingSystem{ + Config: config.ReadConfig(configPath), + } + logger.InitLogger(ts.Config) + ts.Router = gin.Default() // TODO: Add router options (e.g debug) + + // TODO: Add all TestingSystem common components initialization + + return ts +} + +func (ts *TestingSystem) Run() { + addr := ":" + strconv.Itoa(ts.Config.Port) + if ts.Config.Host != nil { + addr = *ts.Config.Host + addr + } + logger.Info("Starting server at " + addr) + err := ts.Router.Run(addr) + if err != nil { + logger.Panic(err.Error()) + } +} diff --git a/lib/config/config.go b/lib/config/config.go new file mode 100644 index 0000000..9d303cf --- /dev/null +++ b/lib/config/config.go @@ -0,0 +1,47 @@ +package config + +import ( + "os" + + "github.com/xorcare/pointer" + "gopkg.in/yaml.v3" +) + +type Config struct { + Port int `yaml:"Port"` + Host *string `yaml:"Host,omitempty"` // leave empty for localhost + + LogPath *string `yaml:"LogPath,omitempty"` + LogLevel *int `yaml:"LogLevel,omitempty"` + + // TODO: Add all components here + + // if instance is set up on server, leave connection empty + MasterConnection *Connection `yaml:"MasterConnection,omitempty"` + StorageConnection *Connection `yaml:"StorageConnection,omitempty"` +} + +func ReadConfig(configPath string) *Config { + content, err := os.ReadFile(configPath) + if err != nil { + panic(err) + } + + config := new(Config) + err = yaml.Unmarshal(content, config) + if err != nil { + panic(err) + } + + fillInConfig(config) + + return config +} + +func fillInConfig(config *Config) { + if config.Host == nil { + config.Host = pointer.String("localhost") + } + + fillInConnections(config) +} diff --git a/lib/config/connection.go b/lib/config/connection.go new file mode 100644 index 0000000..4cf85d2 --- /dev/null +++ b/lib/config/connection.go @@ -0,0 +1,10 @@ +package config + +type Connection struct { + Address string `yaml:"Address"` + // TODO: Add authentification +} + +func fillInConnections(config *Config) { + // TODO: Add auto connection creation if master or storage are set up +} diff --git a/lib/logger/logger.go b/lib/logger/logger.go index 2743d14..73bf13f 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -1,5 +1,80 @@ package logger -func Panic(format string, v ...interface{}) { - // TODO: create logger +import ( + "fmt" + base_log "log" + "os" + "testing_system/lib/config" +) + +const ( + LOG_LEVEL_TRACE = 0 + LOG_LEVEL_DEBUG = 1 + LOG_LEVEL_INFO = 2 + LOG_LEVEL_WARN = 3 + LOG_LEVEL_ERROR = 4 +) + +var ( + logLevel = LOG_LEVEL_DEBUG + log = base_log.New(os.Stdout, "", base_log.Ldate|base_log.Ltime) + logErr = base_log.New(os.Stderr, "", base_log.Ldate|base_log.Ltime) +) + +func Trace(format string, values ...interface{}) { + logPrint(LOG_LEVEL_TRACE, format, values...) +} + +func Debug(format string, values ...any) { + logPrint(LOG_LEVEL_DEBUG, format, values...) +} + +func Info(format string, values ...any) { + logPrint(LOG_LEVEL_INFO, format, values...) +} + +func Warn(format string, values ...any) { + logPrint(LOG_LEVEL_WARN, format, values...) +} + +func Error(format string, values ...any) error { + logPrint(LOG_LEVEL_ERROR, format, values...) + logErr.Printf(logErrPrefix()+format, values...) + return fmt.Errorf(format, values...) +} + +func Panic(format string, value ...any) { + logPrint(LOG_LEVEL_ERROR, format, value...) + logErr.Panicf(format, value...) +} + +func InitLogger(config *config.Config) { + logLevel = LOG_LEVEL_INFO + if config.LogLevel != nil { + logLevel = *config.LogLevel + } + + var logFile, logErrFile *os.File + + if config.LogPath == nil { + logFile = os.Stdout + logErrFile = os.Stderr + } else { + var err error + logFile, err = os.OpenFile(*config.LogPath+".log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0660) + if err != nil { + panic(err) + } + + logErrFile, err = os.OpenFile(*config.LogPath+".err", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0660) + if err != nil { + panic(err) + } + } + + flags := base_log.Ldate | base_log.Ltime + log = base_log.New(logFile, "", flags) + logErr = base_log.New(logErrFile, "", flags) + + Info("Logger is successfully initialized") } diff --git a/lib/logger/utils.go b/lib/logger/utils.go new file mode 100644 index 0000000..614b091 --- /dev/null +++ b/lib/logger/utils.go @@ -0,0 +1,36 @@ +package logger + +import ( + "fmt" + "runtime" +) + +func levelString(level int) string { + switch level { + case LOG_LEVEL_DEBUG: + return "[DEBUG]" + case LOG_LEVEL_INFO: + return "[INFO]" + case LOG_LEVEL_WARN: + return "[WARN]" + case LOG_LEVEL_ERROR: + return "[ERROR]" + default: + return "" + } +} + +func logPrint(level int, format string, value ...any) { + if logLevel <= level { + log.Printf(levelString(level)+" "+format, value...) + } +} + +func logErrPrefix() string { + _, file, line, ok := runtime.Caller(2) + if ok { + return fmt.Sprintf("%s:%d ", file, line) + } else { + return "" + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..b0e71bb --- /dev/null +++ b/main.go @@ -0,0 +1,13 @@ +package main + +import ( + "os" + "testing_system/lib/common" +) + +func main() { + configPath := os.Args[1] + ts := common.InitTestingSystem(configPath) + // TODO: Add all components initialisation + ts.Run() +} From 63b160801089a55c11ff5a07d939af5e7be47c91 Mon Sep 17 00:00:00 2001 From: Philip Gribov Date: Tue, 7 Jan 2025 01:48:19 +1100 Subject: [PATCH 2/5] Add request creation --- go.mod | 11 ++++++----- go.sum | 24 ++++++++++++++---------- lib/common/testing_system.go | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 945271b..ca293fd 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.22rc1 require ( github.com/gin-gonic/gin v1.10.0 + github.com/go-resty/resty/v2 v2.16.2 github.com/lib/pq v1.10.9 github.com/xorcare/pointer v1.2.2 gopkg.in/yaml.v3 v3.0.1 @@ -42,10 +43,10 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index ed323f6..cfefca1 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-resty/resty/v2 v2.16.2 h1:CpRqTjIzq/rweXUt9+GxzzQdlkqMdt8Lm/fuK/CAbAg= +github.com/go-resty/resty/v2 v2.16.2/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= @@ -89,18 +91,20 @@ github.com/xorcare/pointer v1.2.2/go.mod h1:azsKh7oVwYB7C1o8P284fG8MvtErX/F5/dqX golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= diff --git a/lib/common/testing_system.go b/lib/common/testing_system.go index ae4c937..2389911 100644 --- a/lib/common/testing_system.go +++ b/lib/common/testing_system.go @@ -2,6 +2,7 @@ package common import ( "github.com/gin-gonic/gin" + "github.com/go-resty/resty/v2" "strconv" "testing_system/lib/config" "testing_system/lib/logger" @@ -36,3 +37,21 @@ func (ts *TestingSystem) Run() { logger.Panic(err.Error()) } } + +// NewMasterRequest creates new request to master with authentication and address set up. +// master address will be taken from config (it is either to same process, or to the other microservice) +func (ts *TestingSystem) NewMasterRequest() *resty.Request { + // TODO: Add request creation +} + +// NewStorageRequest creates new request to storage with authentication and address set up. +// master address will be taken from config (it is either to same process, or to the other microservice) +func (ts *TestingSystem) NewStorageRequest() *resty.Request { + // TODO: Add request creation +} + +// NewInvokerRequest creates new request to invoker with authentication set up. +// params may be changed depending on how invokers are stored, probably address will be set up by caller +func (ts *TestingSystem) NewInvokerRequest(invokerID string) *resty.Request { + // TODO: Add request creation +} From dca3496006cd7427cc39074573830085433d48fe Mon Sep 17 00:00:00 2001 From: Philip Gribov Date: Fri, 7 Feb 2025 22:57:57 +0300 Subject: [PATCH 3/5] Add invoker base --- {lib => common}/config/config.go | 4 +- {lib => common}/config/connection.go | 2 +- common/config/db.go | 5 + common/config/invoker.go | 23 ++++ common/connectors/invokerconn/structs.go | 19 +++ .../storageconn/resourcetype_string.go | 27 +++++ .../storageconn/storage_connector.go | 25 ++++ common/connectors/storageconn/structs.go | 93 +++++++++++++++ {db => common/db}/db.go | 5 +- {db => common/db}/models/problem.go | 0 common/db/models/problem_config.go | 13 ++ {db => common/db}/models/submission.go | 0 {db => common/db}/models/testing_result.go | 0 {db => common/db}/readme.md | 0 common/testing_system.go | 104 ++++++++++++++++ db/config.go | 5 - go.mod | 5 +- go.sum | 14 ++- invoker/handler.go | 18 +++ invoker/invoker.go | 59 +++++++++ invoker/storage/cache.go | 112 ++++++++++++++++++ invoker/storage/storage.go | 65 ++++++++++ invoker/tester/tester.go | 31 +++++ lib/common/testing_system.go | 57 --------- lib/handler/response.go | 21 ++++ lib/logger/logger.go | 42 ++++--- lib/logger/utils.go | 12 +- main.go | 7 +- 28 files changed, 671 insertions(+), 97 deletions(-) rename {lib => common}/config/config.go (89%) rename {lib => common}/config/connection.go (66%) create mode 100644 common/config/db.go create mode 100644 common/config/invoker.go create mode 100644 common/connectors/invokerconn/structs.go create mode 100644 common/connectors/storageconn/resourcetype_string.go create mode 100644 common/connectors/storageconn/storage_connector.go create mode 100644 common/connectors/storageconn/structs.go rename {db => common/db}/db.go (84%) rename {db => common/db}/models/problem.go (100%) create mode 100644 common/db/models/problem_config.go rename {db => common/db}/models/submission.go (100%) rename {db => common/db}/models/testing_result.go (100%) rename {db => common/db}/readme.md (100%) create mode 100644 common/testing_system.go delete mode 100644 db/config.go create mode 100644 invoker/handler.go create mode 100644 invoker/invoker.go create mode 100644 invoker/storage/cache.go create mode 100644 invoker/storage/storage.go create mode 100644 invoker/tester/tester.go delete mode 100644 lib/common/testing_system.go create mode 100644 lib/handler/response.go diff --git a/lib/config/config.go b/common/config/config.go similarity index 89% rename from lib/config/config.go rename to common/config/config.go index 9d303cf..0c54b17 100644 --- a/lib/config/config.go +++ b/common/config/config.go @@ -14,8 +14,10 @@ type Config struct { LogPath *string `yaml:"LogPath,omitempty"` LogLevel *int `yaml:"LogLevel,omitempty"` - // TODO: Add all components here + Invoker *InvokerConfig `yaml:"Invoker,omitempty"` + // TODO: Add instances here + DB DBConfig `yaml:"DB"` // if instance is set up on server, leave connection empty MasterConnection *Connection `yaml:"MasterConnection,omitempty"` StorageConnection *Connection `yaml:"StorageConnection,omitempty"` diff --git a/lib/config/connection.go b/common/config/connection.go similarity index 66% rename from lib/config/connection.go rename to common/config/connection.go index 4cf85d2..48cce79 100644 --- a/lib/config/connection.go +++ b/common/config/connection.go @@ -6,5 +6,5 @@ type Connection struct { } func fillInConnections(config *Config) { - // TODO: Add auto connection creation if master or storage are set up + // TODO: Add auto connection creation if master or storageconn are set up } diff --git a/common/config/db.go b/common/config/db.go new file mode 100644 index 0000000..03ccab4 --- /dev/null +++ b/common/config/db.go @@ -0,0 +1,5 @@ +package config + +type DBConfig struct { + Dsn string `yaml:"dsn"` +} diff --git a/common/config/invoker.go b/common/config/invoker.go new file mode 100644 index 0000000..fccc954 --- /dev/null +++ b/common/config/invoker.go @@ -0,0 +1,23 @@ +package config + +import "github.com/xorcare/pointer" + +type InvokerConfig struct { + Instances int `yaml:"Instances"` + QueueSize *int `yaml:"QueueSize,omitempty"` // default is 10 + + CacheSize uint64 `yaml:"CacheSize"` + CachePath string `yaml:"CachePath"` +} + +func fillInInvokerConfig(config *InvokerConfig) { + if config.QueueSize == nil { + config.QueueSize = pointer.Int(10) + } + if len(config.CachePath) == 0 { + panic("No invoker cache path specified") + } + if config.CacheSize == 0 { + panic("No invoker cache size specified") + } +} diff --git a/common/connectors/invokerconn/structs.go b/common/connectors/invokerconn/structs.go new file mode 100644 index 0000000..0191fbd --- /dev/null +++ b/common/connectors/invokerconn/structs.go @@ -0,0 +1,19 @@ +package invokerconn + +import ( + models2 "testing_system/common/db/models" +) + +const ( + JobTypeCompile = iota + JobTypeTest +) + +type Job struct { + SubmitID uint `json:"submitID" binding:"required"` + JobType int `json:"jobType" binding:"required"` + Test int `json:"test"` + + Submit *models2.Submission `json:"submit,omitempty"` + Problem *models2.Problem `json:"problem,omitempty"` +} diff --git a/common/connectors/storageconn/resourcetype_string.go b/common/connectors/storageconn/resourcetype_string.go new file mode 100644 index 0000000..5a00e6a --- /dev/null +++ b/common/connectors/storageconn/resourcetype_string.go @@ -0,0 +1,27 @@ +// Code generated by "stringer -type=ResourceType"; DO NOT EDIT. + +package storageconn + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[SourceCode-0] + _ = x[CompiledBinary-1] + _ = x[Test-2] + _ = x[Checker-3] + _ = x[Interactor-4] +} + +const _ResourceType_name = "SourceCodeCompiledBinaryTestCheckerInteractor" + +var _ResourceType_index = [...]uint8{0, 10, 24, 28, 35, 45} + +func (i ResourceType) String() string { + if i < 0 || i >= ResourceType(len(_ResourceType_index)-1) { + return "ResourceType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ResourceType_name[_ResourceType_index[i]:_ResourceType_index[i+1]] +} diff --git a/common/connectors/storageconn/storage_connector.go b/common/connectors/storageconn/storage_connector.go new file mode 100644 index 0000000..e1e10ae --- /dev/null +++ b/common/connectors/storageconn/storage_connector.go @@ -0,0 +1,25 @@ +package storageconn + +import ( + "testing_system/common" +) + +type Connector struct { + // TODO: Add storage data loader +} + +func NewStorageConnector(ts *common.TestingSystem) *Connector { + return nil +} + +func (s *Connector) Download(request Request) *ResponseFiles { + return nil +} + +func (s *Connector) Upload(request Request) *Response { + return nil +} + +func (s *Connector) Delete(request Request) *Response { + return nil +} diff --git a/common/connectors/storageconn/structs.go b/common/connectors/storageconn/structs.go new file mode 100644 index 0000000..a80cf85 --- /dev/null +++ b/common/connectors/storageconn/structs.go @@ -0,0 +1,93 @@ +package storageconn + +//go:generate stringer -type=ResourceType + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "testing_system/lib/logger" +) + +type ResourceType int + +const ( + SourceCode ResourceType = iota + CompiledBinary + Test + Checker + Interactor + // Will be increased +) + +type Request struct { + // Should be always specified + Resource ResourceType `json:"resource"` + + // If resource is part of problem, ProblemID is used + ProblemID uint64 `json:"problemID"` + // If resource is part of submit, SubmitID is used + SubmitID uint64 `json:"submitID"` + // If resource is a test, TestID should be specified + TestID uint64 `json:"testID"` + + // For any download, BaseFolder should be specified. The files with original filenames will be placed there + BaseFolder string `json:"-"` + + // For uploads, FilePath or FilePaths should be specified (depending on whether the resource is single-file or not). + // Filename will be taken from filename inside the path. + FilePath string `json:"-"` + FilePaths []string `json:"-"` +} + +func (s *Request) FillBaseFolder(parent string) { + s.BaseFolder = filepath.Join(parent, s.Resource.String()) + switch s.Resource { + case SourceCode, CompiledBinary: + s.BaseFolder = filepath.Join(s.BaseFolder, strconv.FormatUint(s.SubmitID, 10)) + case Checker, Interactor: + s.BaseFolder = filepath.Join(s.BaseFolder, strconv.FormatUint(s.ProblemID, 10)) + case Test: + s.BaseFolder = filepath.Join(s.BaseFolder, fmt.Sprintf("%d-%d", s.SubmitID, s.TestID)) + default: + logger.Panic("Can not fill base folder for storageconn request of type %s", s.Resource) + } +} + +type Response struct { + R Request + Error error +} + +type ResponseFiles struct { + Response + fileNames []string + Size uint64 +} + +func (r *ResponseFiles) File() string { + return filepath.Join(r.R.BaseFolder, r.fileNames[0]) +} + +func (r *ResponseFiles) Get(fileName string) (string, bool) { + for _, f := range r.fileNames { + if fileName == f { + return filepath.Join(r.R.BaseFolder, r.fileNames[0]), true + } + } + return "", false +} + +func (r *ResponseFiles) CleanUp() { + if r.Error != nil { + return + } + if len(r.R.BaseFolder) == 0 { + return + } + err := os.RemoveAll(r.R.BaseFolder) + if err != nil { + logger.Panic("Can not remove resource folder, error: %s", err.Error()) + } +} diff --git a/db/db.go b/common/db/db.go similarity index 84% rename from db/db.go rename to common/db/db.go index fab4f49..136e67b 100644 --- a/db/db.go +++ b/common/db/db.go @@ -4,10 +4,11 @@ import ( "gorm.io/driver/postgres" "gorm.io/gorm" "log/slog" - "testing_system/db/models" + "testing_system/common/config" + "testing_system/common/db/models" ) -func NewDb(config Config) (*gorm.DB, error) { +func NewDB(config config.DBConfig) (*gorm.DB, error) { db, err := gorm.Open(postgres.Open(config.Dsn), &gorm.Config{}) if err != nil { slog.Error("Can't open database", "dsn", config.Dsn, "err", err) diff --git a/db/models/problem.go b/common/db/models/problem.go similarity index 100% rename from db/models/problem.go rename to common/db/models/problem.go diff --git a/common/db/models/problem_config.go b/common/db/models/problem_config.go new file mode 100644 index 0000000..cd0b663 --- /dev/null +++ b/common/db/models/problem_config.go @@ -0,0 +1,13 @@ +package models + +type ProblemType int + +const ( + ProblemType_ICPC ProblemType = iota + ProblemType_IOI +) + +type ProblemConfig struct { + ProblemType ProblemType + // everything we need here +} diff --git a/db/models/submission.go b/common/db/models/submission.go similarity index 100% rename from db/models/submission.go rename to common/db/models/submission.go diff --git a/db/models/testing_result.go b/common/db/models/testing_result.go similarity index 100% rename from db/models/testing_result.go rename to common/db/models/testing_result.go diff --git a/db/readme.md b/common/db/readme.md similarity index 100% rename from db/readme.md rename to common/db/readme.md diff --git a/common/testing_system.go b/common/testing_system.go new file mode 100644 index 0000000..c0f5849 --- /dev/null +++ b/common/testing_system.go @@ -0,0 +1,104 @@ +package common + +import ( + "context" + "github.com/gin-gonic/gin" + "gorm.io/gorm" + "net/http" + "os/signal" + "strconv" + "sync" + "syscall" + "testing_system/common/config" + "testing_system/common/db" + "testing_system/lib/logger" +) + +type TestingSystem struct { + Config *config.Config + Router *gin.Engine + DB *gorm.DB + + processes []func() + defers []func() + + StopCtx context.Context + stopFunc context.CancelFunc + stopWG sync.WaitGroup +} + +func InitTestingSystem(configPath string) *TestingSystem { + ts := &TestingSystem{ + Config: config.ReadConfig(configPath), + } + logger.InitLogger(ts.Config) + ts.Router = gin.Default() // TODO: Add router options (e.g debug) + + var err error + ts.DB, err = db.NewDB(ts.Config.DB) + if err != nil { + logger.Panic("Can not set up db connection, error: %s", err.Error()) + } + + return ts +} + +func (ts *TestingSystem) AddProcess(f func()) { + ts.processes = append(ts.processes, f) +} + +func (ts *TestingSystem) AddDefer(f func()) { + ts.defers = append(ts.defers, f) +} + +func (ts *TestingSystem) Run() { + var ctx context.Context + var cancel context.CancelFunc + ctx, ts.stopFunc = context.WithCancel(context.Background()) + ts.StopCtx, cancel = signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + for _, process := range ts.processes { + ts.runProcess(process) + } + + ts.runServer() + + ts.stopWG.Wait() + + for _, d := range ts.defers { + d() + } +} + +func (ts *TestingSystem) runServer() { + addr := ":" + strconv.Itoa(ts.Config.Port) + if ts.Config.Host != nil { + addr = *ts.Config.Host + addr + } + logger.Info("Starting server at " + addr) + server := http.Server{ + Addr: addr, + Handler: ts.Router, + } + go func() { + <-ts.StopCtx.Done() + logger.Info("Shutting down server") + server.Shutdown(context.Background()) + }() + server.ListenAndServe() +} + +func (ts *TestingSystem) runProcess(f func()) { + ts.stopWG.Add(1) + defer func() { + v := recover() + if v != nil { + logger.Error("One process got panic, shutting down all processes gracefully") + ts.stopFunc() + } + ts.stopWG.Done() + }() + + f() +} diff --git a/db/config.go b/db/config.go deleted file mode 100644 index f2b9f12..0000000 --- a/db/config.go +++ /dev/null @@ -1,5 +0,0 @@ -package db - -type Config struct { - Dsn string -} diff --git a/go.mod b/go.mod index ca293fd..a29983f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.16.2 github.com/lib/pq v1.10.9 + github.com/otiai10/copy v1.14.1 github.com/xorcare/pointer v1.2.2 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.9 @@ -45,8 +46,8 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/text v0.16.0 // indirect google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index cfefca1..ba5405d 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,11 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -64,6 +67,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -95,12 +99,12 @@ golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= @@ -110,8 +114,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/invoker/handler.go b/invoker/handler.go new file mode 100644 index 0000000..f705209 --- /dev/null +++ b/invoker/handler.go @@ -0,0 +1,18 @@ +package invoker + +import ( + "github.com/gin-gonic/gin" + "net/http" +) + +func (i *Invoker) HandlePing(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "alive": true, + "queueSize:": len(i.Queue), + "maxQueueSize": i.MaxQueueSize, + }) +} + +func (i *Invoker) HandleTest(c *gin.Context) { + // TODO +} diff --git a/invoker/invoker.go b/invoker/invoker.go new file mode 100644 index 0000000..af7e133 --- /dev/null +++ b/invoker/invoker.go @@ -0,0 +1,59 @@ +package invoker + +import ( + "testing_system/common" + "testing_system/common/db/models" + "testing_system/invoker/storage" + "testing_system/invoker/tester" + "testing_system/lib/logger" +) + +type Invoker struct { + TS *common.TestingSystem + + Storage *storage.InvokerStorage + Testers []*tester.Tester + + Queue chan *models.Submission + MaxQueueSize int +} + +func SetupInvoker(ts *common.TestingSystem) { + if ts.Config.Invoker == nil { + logger.Info("Invoker is not configured, skipping invoker start") + return + } + invoker := &Invoker{ + TS: ts, + MaxQueueSize: ts.Config.Invoker.Instances * *ts.Config.Invoker.QueueSize, + } + invoker.Queue = make(chan *models.Submission, invoker.MaxQueueSize) + invoker.Storage = storage.NewInvokerStorage(ts) + + for i := range ts.Config.Invoker.Instances { + invoker.Testers = append(invoker.Testers, tester.NewTester(ts, i, invoker.Storage)) + testerID := i + ts.AddProcess(func() { invoker.RunTesterThread(testerID) }) + ts.AddDefer(invoker.Testers[i].Cleanup) + } + + r := ts.Router.Group("/invoker/") + r.GET("/ping", invoker.HandlePing) + r.POST("/test", invoker.HandleTest) + // TODO +} + +func (i *Invoker) RunTesterThread(id int) { + t := i.Testers[id] + defer t.Cleanup() + logger.Info("Starting invoker %d", id) + for { + select { + case <-i.TS.StopCtx.Done(): + logger.Info("Stopped invoker %d", id) + break + case s := <-i.Queue: + t.Test(s) + } + } +} diff --git a/invoker/storage/cache.go b/invoker/storage/cache.go new file mode 100644 index 0000000..d5c6aa1 --- /dev/null +++ b/invoker/storage/cache.go @@ -0,0 +1,112 @@ +package storage + +import ( + "testing_system/common/connectors/storageconn" + "testing_system/lib/cache" + "testing_system/lib/logger" +) + +type commonCache = cache.LRUSizeCache[cacheKey, storageconn.ResponseFiles] + +type cacheKey struct { + Resource storageconn.ResourceType `json:"resource"` + + // If resource is part of problem, ProblemID is used + ProblemID uint64 `json:"problemID"` + // If resource is part of submit, SubmitID is used + SubmitID uint64 `json:"submitID"` + // If resource is a test, TestID should be specified + TestID uint64 `json:"testID"` +} + +type cacheGetter struct { + Cache *commonCache + keyGen func(vals ...uint64) cacheKey +} + +func (c *cacheGetter) Get(vals ...uint64) (*storageconn.ResponseFiles, error) { + return c.Cache.Get(c.keyGen(vals...)) +} + +func (c *cacheGetter) Lock(vals ...uint64) { + c.Cache.Lock(c.keyGen(vals...)) +} + +func (c *cacheGetter) Unlock(vals ...uint64) error { + return c.Cache.Unlock(c.keyGen(vals...)) +} + +func newSourceCache(commonCache *commonCache) *cacheGetter { + return &cacheGetter{ + Cache: commonCache, + keyGen: func(vals ...uint64) cacheKey { + return submitKeyGen(storageconn.SourceCode, vals) + }, + } +} + +func newBinaryCache(commonCache *commonCache) *cacheGetter { + return &cacheGetter{ + Cache: commonCache, + keyGen: func(vals ...uint64) cacheKey { + return submitKeyGen(storageconn.CompiledBinary, vals) + }, + } +} + +func newCheckerCache(commonCache *commonCache) *cacheGetter { + return &cacheGetter{ + Cache: commonCache, + keyGen: func(vals ...uint64) cacheKey { + return problemIDKeyGen(storageconn.Checker, vals) + }, + } +} + +func newInteractorCache(commonCache *commonCache) *cacheGetter { + return &cacheGetter{ + Cache: commonCache, + keyGen: func(vals ...uint64) cacheKey { + return submitKeyGen(storageconn.Interactor, vals) + }, + } +} + +func newTestCache(commonCache *commonCache) *cacheGetter { + return &cacheGetter{ + Cache: commonCache, + keyGen: func(vals ...uint64) cacheKey { + if len(vals) != 2 { + logger.PanicLevel(2, "wrong usage of storageconn cache, can not get problem tests for ids %v, exactly 2 ids should be passed", vals) + } + key := cacheKey{ + Resource: storageconn.Test, + ProblemID: vals[0], + TestID: vals[1], + } + return key + }, + } +} + +func problemIDKeyGen(resource storageconn.ResourceType, vals []uint64) cacheKey { + if len(vals) != 1 { + logger.PanicLevel(3, "wrong usage of storageconn cache, can not get problem %s for id %v, too many ids passed", resource.String(), vals) + } + key := cacheKey{ + Resource: resource, + ProblemID: vals[0], + } + return key +} + +func submitKeyGen(resource storageconn.ResourceType, vals []uint64) cacheKey { + if len(vals) != 1 { + logger.PanicLevel(3, "wrong usage of storageconn cache, can not get submit %s for id %v, too many ids passed", resource.String(), vals) + } + key := cacheKey{ + Resource: resource, + SubmitID: vals[0], + } + return key +} diff --git a/invoker/storage/storage.go b/invoker/storage/storage.go new file mode 100644 index 0000000..42c379c --- /dev/null +++ b/invoker/storage/storage.go @@ -0,0 +1,65 @@ +package storage + +import ( + "os" + "testing_system/common" + "testing_system/common/connectors/storageconn" + "testing_system/lib/cache" + "testing_system/lib/logger" +) + +type InvokerStorage struct { + ts *common.TestingSystem + storageConnector *storageconn.Connector + + cache *cache.LRUSizeCache[cacheKey, storageconn.ResponseFiles] + + Source *cacheGetter + Binary *cacheGetter + Checker *cacheGetter + Interactor *cacheGetter + Test *cacheGetter +} + +func NewInvokerStorage(ts *common.TestingSystem) *InvokerStorage { + s := &InvokerStorage{ + storageConnector: storageconn.NewStorageConnector(ts), + } + err := os.RemoveAll(ts.Config.Invoker.CachePath) + if err != nil { + logger.Panic(err.Error()) + } + err = os.MkdirAll(ts.Config.Invoker.CachePath, 0775) + if err != nil { + logger.Panic(err.Error()) + } + s.cache = cache.NewLRUSizeCache[cacheKey, storageconn.ResponseFiles]( + ts.Config.Invoker.CacheSize, + s.getFiles, + func(key cacheKey, files *storageconn.ResponseFiles) { + files.CleanUp() + }, + ) + s.Source = newSourceCache(s.cache) + s.Binary = newBinaryCache(s.cache) + s.Checker = newCheckerCache(s.cache) + s.Interactor = newInteractorCache(s.cache) + s.Test = newTestCache(s.cache) + return s +} + +func (s *InvokerStorage) getFiles(key cacheKey) (*storageconn.ResponseFiles, error, uint64) { + request := storageconn.Request{ + Resource: key.Resource, + ProblemID: key.ProblemID, + SubmitID: key.SubmitID, + TestID: key.TestID, + } + request.FillBaseFolder(s.ts.Config.Invoker.CachePath) + files := s.storageConnector.Download(request) + if files.Error != nil { + return nil, files.Error, 0 + } else { + return files, nil, files.Size + } +} diff --git a/invoker/tester/tester.go b/invoker/tester/tester.go new file mode 100644 index 0000000..2a46d55 --- /dev/null +++ b/invoker/tester/tester.go @@ -0,0 +1,31 @@ +package tester + +import ( + "testing_system/common" + "testing_system/common/db/models" + "testing_system/invoker/storage" +) + +type Tester struct { + ID int + Storage *storage.InvokerStorage +} + +func (t *Tester) Test(submit *models.Submission) { + +} + +func (t *Tester) Cleanup() { + +} + +func NewTester( + ts *common.TestingSystem, + id int, + storage *storage.InvokerStorage, +) *Tester { + return &Tester{ + ID: id, + Storage: storage, + } +} diff --git a/lib/common/testing_system.go b/lib/common/testing_system.go deleted file mode 100644 index 2389911..0000000 --- a/lib/common/testing_system.go +++ /dev/null @@ -1,57 +0,0 @@ -package common - -import ( - "github.com/gin-gonic/gin" - "github.com/go-resty/resty/v2" - "strconv" - "testing_system/lib/config" - "testing_system/lib/logger" -) - -type TestingSystem struct { - Config *config.Config - Router *gin.Engine - // TODO: Add common params for all testing system components -} - -func InitTestingSystem(configPath string) *TestingSystem { - ts := &TestingSystem{ - Config: config.ReadConfig(configPath), - } - logger.InitLogger(ts.Config) - ts.Router = gin.Default() // TODO: Add router options (e.g debug) - - // TODO: Add all TestingSystem common components initialization - - return ts -} - -func (ts *TestingSystem) Run() { - addr := ":" + strconv.Itoa(ts.Config.Port) - if ts.Config.Host != nil { - addr = *ts.Config.Host + addr - } - logger.Info("Starting server at " + addr) - err := ts.Router.Run(addr) - if err != nil { - logger.Panic(err.Error()) - } -} - -// NewMasterRequest creates new request to master with authentication and address set up. -// master address will be taken from config (it is either to same process, or to the other microservice) -func (ts *TestingSystem) NewMasterRequest() *resty.Request { - // TODO: Add request creation -} - -// NewStorageRequest creates new request to storage with authentication and address set up. -// master address will be taken from config (it is either to same process, or to the other microservice) -func (ts *TestingSystem) NewStorageRequest() *resty.Request { - // TODO: Add request creation -} - -// NewInvokerRequest creates new request to invoker with authentication set up. -// params may be changed depending on how invokers are stored, probably address will be set up by caller -func (ts *TestingSystem) NewInvokerRequest(invokerID string) *resty.Request { - // TODO: Add request creation -} diff --git a/lib/handler/response.go b/lib/handler/response.go new file mode 100644 index 0000000..ef0ec2b --- /dev/null +++ b/lib/handler/response.go @@ -0,0 +1,21 @@ +package handler + +import ( + "fmt" + "github.com/gin-gonic/gin" + "net/http" +) + +func RespOK(c *gin.Context, data gin.H) { + c.JSON(http.StatusOK, gin.H{ + "ok": true, + "response": data, + }) +} + +func RespErr(c *gin.Context, code int, errf string, values ...interface{}) { + c.JSON(code, gin.H{ + "ok": false, + "error": fmt.Sprintf(errf, values...), + }) +} diff --git a/lib/logger/logger.go b/lib/logger/logger.go index 73bf13f..ad2f35b 100644 --- a/lib/logger/logger.go +++ b/lib/logger/logger.go @@ -4,52 +4,60 @@ import ( "fmt" base_log "log" "os" - "testing_system/lib/config" + "testing_system/common/config" ) const ( - LOG_LEVEL_TRACE = 0 - LOG_LEVEL_DEBUG = 1 - LOG_LEVEL_INFO = 2 - LOG_LEVEL_WARN = 3 - LOG_LEVEL_ERROR = 4 + LogLevelTrace = 0 + LogLevelDebug = 1 + LogLevelInfo = 2 + LogLevelWarn = 3 + LogLevelError = 4 ) var ( - logLevel = LOG_LEVEL_DEBUG + logLevel = LogLevelDebug log = base_log.New(os.Stdout, "", base_log.Ldate|base_log.Ltime) logErr = base_log.New(os.Stderr, "", base_log.Ldate|base_log.Ltime) ) func Trace(format string, values ...interface{}) { - logPrint(LOG_LEVEL_TRACE, format, values...) + logPrint(LogLevelTrace, format, values...) } func Debug(format string, values ...any) { - logPrint(LOG_LEVEL_DEBUG, format, values...) + logPrint(LogLevelDebug, format, values...) } func Info(format string, values ...any) { - logPrint(LOG_LEVEL_INFO, format, values...) + logPrint(LogLevelInfo, format, values...) } func Warn(format string, values ...any) { - logPrint(LOG_LEVEL_WARN, format, values...) + logPrint(LogLevelWarn, format, values...) } func Error(format string, values ...any) error { - logPrint(LOG_LEVEL_ERROR, format, values...) - logErr.Printf(logErrPrefix()+format, values...) + logPrint(LogLevelError, format, values...) + logErr.Printf(logErrPrefix(0)+format, values...) return fmt.Errorf(format, values...) } -func Panic(format string, value ...any) { - logPrint(LOG_LEVEL_ERROR, format, value...) - logErr.Panicf(format, value...) +func Panic(format string, values ...any) { + logPrint(LogLevelError, format, values...) + logErr.Printf(logErrPrefix(0)+format, values...) + panic(fmt.Errorf(format, values...)) +} + +// PanicLevel will output to log the code line which is reachable by level call depth. May be applied in some library code +func PanicLevel(level int, format string, values ...any) { + logPrint(LogLevelError, format, values...) + logErr.Printf(logErrPrefix(level)+format, values...) + panic(fmt.Errorf(format, values...)) } func InitLogger(config *config.Config) { - logLevel = LOG_LEVEL_INFO + logLevel = LogLevelInfo if config.LogLevel != nil { logLevel = *config.LogLevel } diff --git a/lib/logger/utils.go b/lib/logger/utils.go index 614b091..e8c790f 100644 --- a/lib/logger/utils.go +++ b/lib/logger/utils.go @@ -7,13 +7,13 @@ import ( func levelString(level int) string { switch level { - case LOG_LEVEL_DEBUG: + case LogLevelDebug: return "[DEBUG]" - case LOG_LEVEL_INFO: + case LogLevelInfo: return "[INFO]" - case LOG_LEVEL_WARN: + case LogLevelWarn: return "[WARN]" - case LOG_LEVEL_ERROR: + case LogLevelError: return "[ERROR]" default: return "" @@ -26,8 +26,8 @@ func logPrint(level int, format string, value ...any) { } } -func logErrPrefix() string { - _, file, line, ok := runtime.Caller(2) +func logErrPrefix(level int) string { + _, file, line, ok := runtime.Caller(level + 2) if ok { return fmt.Sprintf("%s:%d ", file, line) } else { diff --git a/main.go b/main.go index b0e71bb..fb8d169 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,15 @@ package main import ( "os" - "testing_system/lib/common" + "testing_system/common" + "testing_system/invoker" ) func main() { configPath := os.Args[1] ts := common.InitTestingSystem(configPath) - // TODO: Add all components initialisation + if ts.Config.Invoker != nil { + invoker.SetupInvoker(ts) + } ts.Run() } From b60c62bdbb2164ba76df27b779289f0ce4327631 Mon Sep 17 00:00:00 2001 From: Philip Gribov Date: Fri, 7 Feb 2025 23:46:29 +0300 Subject: [PATCH 4/5] Fixes and notes --- common/connectors/invokerconn/structs.go | 2 ++ common/db/models/problem_config.go | 13 ------------- invoker/invoker.go | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) delete mode 100644 common/db/models/problem_config.go diff --git a/common/connectors/invokerconn/structs.go b/common/connectors/invokerconn/structs.go index 0191fbd..d31214d 100644 --- a/common/connectors/invokerconn/structs.go +++ b/common/connectors/invokerconn/structs.go @@ -4,6 +4,8 @@ import ( models2 "testing_system/common/db/models" ) +// This will be changed in later commits + const ( JobTypeCompile = iota JobTypeTest diff --git a/common/db/models/problem_config.go b/common/db/models/problem_config.go deleted file mode 100644 index cd0b663..0000000 --- a/common/db/models/problem_config.go +++ /dev/null @@ -1,13 +0,0 @@ -package models - -type ProblemType int - -const ( - ProblemType_ICPC ProblemType = iota - ProblemType_IOI -) - -type ProblemConfig struct { - ProblemType ProblemType - // everything we need here -} diff --git a/invoker/invoker.go b/invoker/invoker.go index af7e133..25d4ca4 100644 --- a/invoker/invoker.go +++ b/invoker/invoker.go @@ -14,7 +14,7 @@ type Invoker struct { Storage *storage.InvokerStorage Testers []*tester.Tester - Queue chan *models.Submission + Queue chan *models.Submission // This will be changed in later commits MaxQueueSize int } From b183c0a73346b226070c30cb787fc761cf3b2a9a Mon Sep 17 00:00:00 2001 From: Philip Gribov Date: Thu, 13 Feb 2025 19:58:37 +0300 Subject: [PATCH 5/5] Fixes --- common/connectors/invokerconn/structs.go | 6 +++--- common/connectors/storageconn/structs.go | 9 ++++++--- invoker/storage/cache.go | 10 ++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/common/connectors/invokerconn/structs.go b/common/connectors/invokerconn/structs.go index d31214d..4f2b1b9 100644 --- a/common/connectors/invokerconn/structs.go +++ b/common/connectors/invokerconn/structs.go @@ -1,7 +1,7 @@ package invokerconn import ( - models2 "testing_system/common/db/models" + "testing_system/common/db/models" ) // This will be changed in later commits @@ -16,6 +16,6 @@ type Job struct { JobType int `json:"jobType" binding:"required"` Test int `json:"test"` - Submit *models2.Submission `json:"submit,omitempty"` - Problem *models2.Problem `json:"problem,omitempty"` + Submit *models.Submission `json:"submit,omitempty"` + Problem *models.Problem `json:"problem,omitempty"` } diff --git a/common/connectors/storageconn/structs.go b/common/connectors/storageconn/structs.go index a80cf85..4e9d716 100644 --- a/common/connectors/storageconn/structs.go +++ b/common/connectors/storageconn/structs.go @@ -66,14 +66,17 @@ type ResponseFiles struct { Size uint64 } -func (r *ResponseFiles) File() string { - return filepath.Join(r.R.BaseFolder, r.fileNames[0]) +func (r *ResponseFiles) File() (string, bool) { + if len(r.fileNames) == 0 { + return "", false + } + return filepath.Join(r.R.BaseFolder, r.fileNames[0]), true } func (r *ResponseFiles) Get(fileName string) (string, bool) { for _, f := range r.fileNames { if fileName == f { - return filepath.Join(r.R.BaseFolder, r.fileNames[0]), true + return filepath.Join(r.R.BaseFolder, f), true } } return "", false diff --git a/invoker/storage/cache.go b/invoker/storage/cache.go index d5c6aa1..9f9709f 100644 --- a/invoker/storage/cache.go +++ b/invoker/storage/cache.go @@ -6,6 +6,16 @@ import ( "testing_system/lib/logger" ) +// The main idea of cache is following: We have one cache of specific size to hold all types of files. +// However, we have multiple file types so we must have different getter for each file. +// +// So we have single LRUSizeCache that holds all file types. Key for this cache is cacheKey, the internal struct with which we can determine the file type and it's keys. +// +// To access cache we have cacheGetter structs. Each cacheGetter responds for single file type. +// Each cacheGetter accepts any number of uint64 that are transformed to cacheKey struct using cacheGetter.keyGen func. +// +// So to access files, we call methods of cacheGetter, that transforms our request to request for LRUSizeCache and LRUSizeCache then does all the cache work. + type commonCache = cache.LRUSizeCache[cacheKey, storageconn.ResponseFiles] type cacheKey struct {