From bed33a81e12c72dadc0f90ff2b696d3782605bf6 Mon Sep 17 00:00:00 2001 From: Guangyu Xu Date: Fri, 5 Jan 2024 17:47:01 +0000 Subject: [PATCH] anawareness runtime prefetch 1. add prefetchlist store prefetchlist storage service. 2. Modify the optimizer to publish the access file list as a prefetchlist to the storage service when obtaining it. 3. modify http server add lru algo 4. use echo web framework 5. modify based on comments --- cmd/optimizer-nri-plugin/main.go | 143 +++++++++++++++++++++--- go.mod | 23 ++-- go.sum | 41 ++++--- misc/example/optimizer-nri-plugin.conf | 4 +- pkg/fanotify/fanotify.go | 45 ++++---- tools/prefetch-distribution/main.go | 149 +++++++++++++++++++++++++ 6 files changed, 347 insertions(+), 58 deletions(-) create mode 100644 tools/prefetch-distribution/main.go diff --git a/cmd/optimizer-nri-plugin/main.go b/cmd/optimizer-nri-plugin/main.go index 13c51dffb9..884a169583 100644 --- a/cmd/optimizer-nri-plugin/main.go +++ b/cmd/optimizer-nri-plugin/main.go @@ -7,10 +7,13 @@ package main import ( + "bytes" "context" + "encoding/json" "fmt" "io" "log/syslog" + "net/http" "os" "path/filepath" "strings" @@ -38,11 +41,12 @@ const ( type PluginConfig struct { Events []string `toml:"events"` - ServerPath string `toml:"server_path"` - PersistDir string `toml:"persist_dir"` - Readable bool `toml:"readable"` - Timeout int `toml:"timeout"` - Overwrite bool `toml:"overwrite"` + ServerPath string `toml:"server_path"` + PersistDir string `toml:"persist_dir"` + Readable bool `toml:"readable"` + Timeout int `toml:"timeout"` + Overwrite bool `toml:"overwrite"` + PrefetchDistributionEndpoint string `toml:"prefetch_distribution_endpoint"` } type PluginArgs struct { @@ -104,6 +108,11 @@ func buildFlags(args *PluginArgs) []cli.Flag { Usage: "whether to overwrite the existed persistent files", Destination: &args.Config.Overwrite, }, + &cli.StringFlag{ + Name: "prefetch-distribution-endpoint", + Usage: "The service endpoint of prefetch distribution, for example: http://localhost:1323/api/v1/prefetch/upload", + Destination: &args.Config.PrefetchDistributionEndpoint, + }, } } @@ -129,7 +138,8 @@ var ( ) const ( - imageNameLabel = "io.kubernetes.cri.image-name" + imageNameLabel = "io.kubernetes.cri.image-name" + containerNameLabel = "io.kubernetes.cri.container-name" ) func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, error) { @@ -156,11 +166,26 @@ func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, err return p.mask, nil } +type PrefetchFile struct { + Path string +} + +type CacheItem struct { + ImageName string + ContainerName string + PrefetchFiles []PrefetchFile +} + +type Cache struct { + Items map[string]*CacheItem +} + func (p *plugin) StartContainer(_ *api.PodSandbox, container *api.Container) error { - dir, imageName, err := GetImageName(container.Annotations) + dir, imageName, imageRepo, err := GetImageName(container.Annotations) if err != nil { return err } + containerName := container.Annotations[containerNameLabel] persistDir := filepath.Join(cfg.PersistDir, dir) if err := os.MkdirAll(persistDir, os.ModePerm); err != nil { @@ -172,37 +197,127 @@ func (p *plugin) StartContainer(_ *api.PodSandbox, container *api.Container) err persistFile = fmt.Sprintf("%s.timeout%ds", persistFile, cfg.Timeout) } - fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter) + var hasSentPrefetchList = false + + fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter, containerName, hasSentPrefetchList) if err := fanotifyServer.RunServer(); err != nil { return err } + go func() { + time.Sleep(10 * time.Minute) + fanotifyServer.Mu.Lock() + if !fanotifyServer.IsSent { + data, err := getPrefetchList(persistFile) + if err != nil { + log.WithError(err).Error("error reading file") + } + if err = sendToServer(imageRepo, containerName, cfg.PrefetchDistributionEndpoint, data); err != nil { + log.WithError(err).Error("failed to send prefetch to http server") + } + fanotifyServer.IsSent = true + } + fanotifyServer.Mu.Unlock() + }() + globalFanotifyServer[imageName] = fanotifyServer return nil } +func sendToServer(imageName, containerName, serverURL string, data []byte) error { + filePaths := strings.Split(string(data), "\n") + + var prefetchFiles []PrefetchFile + for _, path := range filePaths { + if path != "" { + prefetchFiles = append(prefetchFiles, PrefetchFile{Path: path}) + } + } + + item := CacheItem{ + ImageName: imageName, + ContainerName: containerName, + PrefetchFiles: prefetchFiles, + } + + err := postRequest(item, serverURL) + if err != nil { + return errors.Wrap(err, "error uploading to server") + } + + return nil +} + +func postRequest(item CacheItem, endpoint string) error { + data, err := json.Marshal(item) + if err != nil { + return err + } + + resp, err := http.Post(endpoint, "application/json", bytes.NewBuffer(data)) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errors.Wrap(fmt.Errorf("server returned a non-OK status code: %d", resp.StatusCode), "HTTP Status Error") + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "failed to read response body") + } + + log.Info("Server Response:", string(body)) + + return nil +} + +func getPrefetchList(prefetchListPath string) ([]byte, error) { + data, err := os.ReadFile(prefetchListPath) + if err != nil { + return nil, err + } + return data, nil +} + func (p *plugin) StopContainer(_ *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) { var update = []*api.ContainerUpdate{} - _, imageName, err := GetImageName(container.Annotations) + _, imageName, imageRepo, err := GetImageName(container.Annotations) if err != nil { return update, err } + if fanotifyServer, ok := globalFanotifyServer[imageName]; ok { - fanotifyServer.StopServer() + fanotifyServer.Mu.Lock() + if !fanotifyServer.IsSent { + data, err := getPrefetchList(fanotifyServer.PersistFile) + if err != nil { + return update, err + } + if err = sendToServer(imageRepo, fanotifyServer.ContainerName, cfg.PrefetchDistributionEndpoint, data); err != nil { + log.WithError(err).Error("failed to send prefetch to http server") + } + fanotifyServer.IsSent = true + + fanotifyServer.StopServer() + } + fanotifyServer.Mu.Unlock() } else { return nil, errors.New("can not find fanotify server for container image " + imageName) } - return update, nil } -func GetImageName(annotations map[string]string) (string, string, error) { +func GetImageName(annotations map[string]string) (string, string, string, error) { named, err := docker.ParseDockerRef(annotations[imageNameLabel]) if err != nil { - return "", "", err + return "", "", "", err } + imageRepo := docker.Named.String(named) nameTagged := named.(docker.NamedTagged) repo := docker.Path(nameTagged) @@ -211,7 +326,7 @@ func GetImageName(annotations map[string]string) (string, string, error) { imageName := image + ":" + nameTagged.Tag() - return dir, imageName, nil + return dir, imageName, imageRepo, nil } func (p *plugin) onClose() { diff --git a/go.mod b/go.mod index 93af640cef..a2ba45c3f9 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.2 github.com/imdario/mergo v0.3.13 github.com/klauspost/compress v1.16.0 + github.com/labstack/echo/v4 v4.11.4 github.com/moby/locker v1.0.1 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/opencontainers/go-digest v1.0.0 @@ -38,13 +39,13 @@ require ( github.com/prometheus/client_model v0.3.0 github.com/rs/xid v1.4.0 github.com/sirupsen/logrus v1.9.0 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.0 go.etcd.io/bbolt v1.3.7 golang.org/x/exp v0.0.0-20231006140011-7918f672742d - golang.org/x/net v0.17.0 + golang.org/x/net v0.19.0 golang.org/x/sync v0.4.0 - golang.org/x/sys v0.13.0 + golang.org/x/sys v0.15.0 google.golang.org/grpc v1.59.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gotest.tools v2.2.0+incompatible @@ -55,6 +56,12 @@ require ( k8s.io/utils v0.0.0-20230726121419-3b25d923346b ) +require ( + github.com/labstack/gommon v0.4.2 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect +) + require ( github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect @@ -112,7 +119,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -137,12 +144,12 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect diff --git a/go.sum b/go.sum index 5b08546849..94ff742b47 100644 --- a/go.sum +++ b/go.sum @@ -236,6 +236,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8= +github.com/labstack/echo/v4 v4.11.4/go.mod h1:noh7EvLwqDsmh/X/HWKPUl1AjzJrhyptRyEbQJfxen8= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -244,8 +248,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= @@ -333,13 +337,17 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -367,8 +375,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= @@ -395,8 +403,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= @@ -432,20 +440,21 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/misc/example/optimizer-nri-plugin.conf b/misc/example/optimizer-nri-plugin.conf index 7b6a561eb1..0080086bb1 100644 --- a/misc/example/optimizer-nri-plugin.conf +++ b/misc/example/optimizer-nri-plugin.conf @@ -10,4 +10,6 @@ timeout = 0 overwrite = false # The events that containerd subscribes to. # Do not change this element. -events = [ "StartContainer", "StopContainer" ] \ No newline at end of file +events = [ "StartContainer", "StopContainer" ] +# The service endpoint of prefetch distribution. +prefetch_distribution_endpoint = "http://localhost:1323/api/v1/prefetch/upload" \ No newline at end of file diff --git a/pkg/fanotify/fanotify.go b/pkg/fanotify/fanotify.go index 7d272cea42..1fd0c425ca 100644 --- a/pkg/fanotify/fanotify.go +++ b/pkg/fanotify/fanotify.go @@ -14,6 +14,7 @@ import ( "log/syslog" "os" "os/exec" + "sync" "syscall" "time" @@ -24,28 +25,33 @@ import ( ) type Server struct { - BinaryPath string - ContainerPid uint32 - ImageName string - PersistFile string - Readable bool - Overwrite bool - Timeout time.Duration - Client *conn.Client - Cmd *exec.Cmd - LogWriter *syslog.Writer + BinaryPath string + ContainerPid uint32 + ImageName string + PersistFile string + Readable bool + Overwrite bool + Timeout time.Duration + Client *conn.Client + Cmd *exec.Cmd + LogWriter *syslog.Writer + ContainerName string + IsSent bool + Mu sync.Mutex } -func NewServer(binaryPath string, containerPid uint32, imageName string, persistFile string, readable bool, overwrite bool, timeout time.Duration, logWriter *syslog.Writer) *Server { +func NewServer(binaryPath string, containerPid uint32, imageName string, persistFile string, readable bool, overwrite bool, timeout time.Duration, logWriter *syslog.Writer, containerName string, hasSentPrefetchList bool) *Server { return &Server{ - BinaryPath: binaryPath, - ContainerPid: containerPid, - ImageName: imageName, - PersistFile: persistFile, - Readable: readable, - Overwrite: overwrite, - Timeout: timeout, - LogWriter: logWriter, + BinaryPath: binaryPath, + ContainerPid: containerPid, + ImageName: imageName, + PersistFile: persistFile, + Readable: readable, + Overwrite: overwrite, + Timeout: timeout, + LogWriter: logWriter, + ContainerName: containerName, + IsSent: hasSentPrefetchList, } } @@ -137,6 +143,7 @@ func (fserver *Server) RunReceiver() error { return errors.Wrapf(err, "failed to write csv") } csvWriter.Flush() + } } diff --git a/tools/prefetch-distribution/main.go b/tools/prefetch-distribution/main.go new file mode 100644 index 0000000000..3e168d1a78 --- /dev/null +++ b/tools/prefetch-distribution/main.go @@ -0,0 +1,149 @@ +/* +* Copyright (c) 2023. Nydus Developers. All rights reserved. +* +* SPDX-License-Identifier: Apache-2.0 + */ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "sync" + + "github.com/labstack/echo/v4" + "github.com/pkg/errors" +) + +type PrefetchFile struct { + Path string +} + +type LRUItem struct { + CacheItem *CacheItem + Prev *LRUItem + Next *LRUItem +} + +type CacheItem struct { + ImageName string + ContainerName string + PrefetchFiles []PrefetchFile +} + +type Cache struct { + Items map[string]*LRUItem + Head *LRUItem + Tail *LRUItem + MaxSize int + mutex sync.Mutex +} + +func (cache *Cache) key(imageName, containerName string) string { + return fmt.Sprintf("%s,%s", imageName, containerName) +} + +func (cache *Cache) Get(imageName, containerName string) (*CacheItem, error) { + cache.mutex.Lock() + defer cache.mutex.Unlock() + + key := cache.key(imageName, containerName) + lruItem, exists := cache.Items[key] + if !exists { + return nil, errors.New("item not found in cache") + } + + cache.removeNode(lruItem) + cache.addtoHead(cache.Head, lruItem) + + return lruItem.CacheItem, nil +} + +func (cache *Cache) Set(item CacheItem) { + cache.mutex.Lock() + defer cache.mutex.Unlock() + + key := cache.key(item.ImageName, item.ContainerName) + if lruItem, exists := cache.Items[key]; exists { + cache.removeNode(lruItem) + cache.addtoHead(cache.Head, lruItem) + + lruItem.CacheItem = &item + } else { + newLRUItem := &LRUItem{ + CacheItem: &item, + } + + cache.addtoHead(cache.Head, newLRUItem) + + cache.Items[key] = newLRUItem + + if len(cache.Items) > cache.MaxSize { + tail := cache.Tail.Prev + cache.removeNode(tail) + delete(cache.Items, cache.key(tail.CacheItem.ImageName, tail.CacheItem.ContainerName)) + } + } +} + +func (cache *Cache) removeNode(lruItem *LRUItem) { + lruItem.Prev.Next = lruItem.Next + lruItem.Next.Prev = lruItem.Prev +} + +func (cache *Cache) addtoHead(head, lruItem *LRUItem) { + next := head.Next + head.Next = lruItem + lruItem.Prev = head + lruItem.Next = next + next.Prev = lruItem +} + +var serverCache Cache + +func uploadHandler(c echo.Context) error { + var item CacheItem + body, err := io.ReadAll(c.Request().Body) + if err != nil { + return c.String(http.StatusBadRequest, "Failed to read request body") + } + err = json.Unmarshal(body, &item) + if err != nil { + return c.String(http.StatusBadRequest, "Invalid request payload") + } + + serverCache.Set(item) + return c.String(http.StatusOK, fmt.Sprintf("Uploaded CacheItem for %s, %s successfully", item.ImageName, item.ContainerName)) +} + +func downloadHandler(c echo.Context) error { + imageName := c.QueryParam("imageName") + containerName := c.QueryParam("containerName") + + item, err := serverCache.Get(imageName, containerName) + if err != nil { + return c.String(http.StatusNotFound, "CacheItem not found") + } + + return c.JSON(http.StatusOK, item) + +} + +func main() { + head, tail := &LRUItem{}, &LRUItem{} + head.Next = tail + tail.Prev = head + serverCache = Cache{ + Items: make(map[string]*LRUItem), + Head: head, + Tail: tail, + MaxSize: 1000, + } + + e := echo.New() + e.POST("/api/v1/prefetch/upload", uploadHandler) + e.GET("/api/v1/prefetch/download", downloadHandler) + + e.Logger.Fatal(e.Start(":1323")) +}