From 1e0cf1514442bf88e55e9ab4c38ddf3fd176b8b4 Mon Sep 17 00:00:00 2001 From: Thijs Metsch Date: Wed, 13 Aug 2025 15:59:25 +0200 Subject: [PATCH] Release 0.4.0 - Security patches. - Dependency Updates. - Versions bump for K8s 1.33 and Go 1.24. - Added power & energy actuator. - Refactored RDT actuator. - Tweaks to vertical and horizontal actuators. - Changes to CRDs to support tolerances & active management. - Reworked logging capabilities. Co-authored-by: tmetsch tmetsch@users.noreply.github.com Co-authored-by: togashidm togashidm@users.noreply.github.com Co-authored-by: ddib ddib@users.noreply.github.com Co-authored-by: davecremins davecremins@users.noreply.github.com --- .github/workflows/test-build.yml | 4 +- Dockerfile | 2 +- Makefile | 12 +- README.md | 1 - artefacts/deploy/manifest.yaml | 2 +- artefacts/examples/default_profiles.yaml | 12 +- artefacts/examples/example_deployment.yaml | 2 +- artefacts/intents_crds_v1alpha1.yaml | 19 +- cmd/main.go | 31 +- docs/actuators.md | 4 + docs/fig/intents_objectives_kpis.png | Bin 17366 -> 16959 bytes docs/fig/intents_objectives_kpis.puml | 5 +- docs/getting_started.md | 32 +- docs/planner_logs.md | 200 ++++++ go.mod | 67 +- go.sum | 164 ++--- pkg/api/intents/v1alpha1/types.go | 9 +- .../plugins/v1alpha1/actuator_client_stub.go | 5 +- .../plugins/v1alpha1/actuator_plugin_stub.go | 4 + pkg/api/plugins/v1alpha1/plugin_manager.go | 1 + .../plugins/v1alpha1/plugin_manager_test.go | 2 +- pkg/api/plugins/v1alpha1/protobufs/api.pb.go | 354 +++++----- pkg/api/plugins/v1alpha1/protobufs/api.proto | 2 + .../plugins/v1alpha1/protobufs/api_grpc.pb.go | 42 +- pkg/common/config.go | 32 + pkg/common/config_test.go | 200 +++++- pkg/common/types.go | 19 +- pkg/common/types_test.go | 14 +- pkg/controller/intent_controller.go | 7 +- pkg/controller/intent_monitor.go | 27 +- pkg/controller/pod_monitor.go | 12 +- pkg/controller/profile_monitor.go | 18 +- pkg/controller/profile_monitor_test.go | 21 +- pkg/controller/state_helper.go | 1 + pkg/controller/state_helper_test.go | 4 +- pkg/planner/actuators/energy/README.md | 24 + .../actuators/energy/analytics/analytics.py | 242 +++++++ .../actuators/energy/analytics/predict.py | 160 +++++ .../energy/analytics/test_analytics.py | 6 + .../energy/analytics/test_predict.py | 63 ++ pkg/planner/actuators/energy/power.go | 394 +++++++++++ pkg/planner/actuators/energy/power_test.go | 513 +++++++++++++++ pkg/planner/actuators/platform/README.md | 21 + .../platform/analytics/rdt_effect.py | 283 ++++++++ .../platform/analytics/rdt_predict.py | 136 ++++ .../platform/analytics/test_analyze.py | 6 + .../platform/analytics/test_predict.py | 81 +++ pkg/planner/actuators/platform/rdt.go | 171 +++-- .../actuators/platform/rdt_sequence.png | Bin 94051 -> 0 bytes .../actuators/platform/rdt_sequence.puml | 69 -- pkg/planner/actuators/platform/rdt_test.go | 234 +++++-- .../scaling/analytics/cpu_rightsizing.py | 5 +- pkg/planner/actuators/scaling/cpu_scale.go | 210 +++--- .../actuators/scaling/cpu_scale_test.go | 170 +++-- pkg/planner/actuators/scaling/rm_pod_test.go | 60 +- pkg/planner/actuators/scaling/scale_out.go | 45 +- .../actuators/scaling/scale_out_test.go | 86 +-- pkg/planner/astar/astar_planner_test.go | 16 +- pkg/tests/energy_effieciency_test.go | 72 ++ pkg/tests/full_framework_test.go | 104 ++- pkg/tests/full_planner_test.go | 6 +- pkg/tests/traces/cpu_scale.json | 1 + pkg/tests/traces/defaults.json | 10 +- pkg/tests/traces/rdt.json | 7 + pkg/tests/traces/trace_0/effects.json | 24 +- pkg/tests/traces/trace_0/events.json | 2 +- pkg/tests/traces/trace_1/effects.json | 410 ++++++------ pkg/tests/traces/trace_2/effects.json | 30 + pkg/tests/traces/trace_2/events.json | 229 +++++++ pkg/tests/traces/trace_rdt/effects.json | 35 + pkg/tests/traces/trace_rdt/events.json | 615 ++++++++++++++++++ pkg/tests/traces/trace_rdt/p95_rdt.csv | 94 +++ pkg/tests/traces/trace_rdt/predict.py | 54 ++ plugins/cpu_scale/Dockerfile | 10 +- plugins/cpu_scale/cmd/cpu_scale.go | 8 +- plugins/cpu_scale/cmd/cpu_scale_test.go | 121 ++-- .../cpu_scale/cpu-scale-actuator-plugin.yaml | 3 +- plugins/energy/Dockerfile | 31 + plugins/energy/cmd/energy.go | 99 +++ plugins/energy/cmd/energy_test.go | 122 ++++ plugins/energy/energy-actuator-plugin.yaml | 99 +++ plugins/rdt/Dockerfile | 14 +- plugins/rdt/cmd/rdt_test.go | 15 +- plugins/rdt/rdt-actuator-plugin.yaml | 7 +- plugins/rm_pod/Dockerfile | 2 +- plugins/rm_pod/rmpod-actuator-plugin.yaml | 3 +- plugins/scale_out/Dockerfile | 10 +- .../scale_out/scaleout-actuator-plugin.yaml | 3 +- 88 files changed, 5378 insertions(+), 1188 deletions(-) create mode 100644 docs/planner_logs.md create mode 100644 pkg/planner/actuators/energy/README.md create mode 100644 pkg/planner/actuators/energy/analytics/analytics.py create mode 100644 pkg/planner/actuators/energy/analytics/predict.py create mode 100644 pkg/planner/actuators/energy/analytics/test_analytics.py create mode 100644 pkg/planner/actuators/energy/analytics/test_predict.py create mode 100644 pkg/planner/actuators/energy/power.go create mode 100644 pkg/planner/actuators/energy/power_test.go create mode 100644 pkg/planner/actuators/platform/README.md create mode 100644 pkg/planner/actuators/platform/analytics/rdt_effect.py create mode 100644 pkg/planner/actuators/platform/analytics/rdt_predict.py create mode 100644 pkg/planner/actuators/platform/analytics/test_analyze.py create mode 100644 pkg/planner/actuators/platform/analytics/test_predict.py delete mode 100644 pkg/planner/actuators/platform/rdt_sequence.png delete mode 100644 pkg/planner/actuators/platform/rdt_sequence.puml create mode 100644 pkg/tests/energy_effieciency_test.go create mode 100644 pkg/tests/traces/rdt.json create mode 100644 pkg/tests/traces/trace_2/effects.json create mode 100644 pkg/tests/traces/trace_2/events.json create mode 100644 pkg/tests/traces/trace_rdt/effects.json create mode 100644 pkg/tests/traces/trace_rdt/events.json create mode 100644 pkg/tests/traces/trace_rdt/p95_rdt.csv create mode 100644 pkg/tests/traces/trace_rdt/predict.py create mode 100644 plugins/energy/Dockerfile create mode 100644 plugins/energy/cmd/energy.go create mode 100644 plugins/energy/cmd/energy_test.go create mode 100644 plugins/energy/energy-actuator-plugin.yaml diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 77b2a34..7e1c960 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -29,6 +29,8 @@ jobs: uses: actions/setup-go@v5 with: go-version-file: 'go.mod' + - name: List open ports and processes + run: sudo lsof -i -P -n | grep LISTEN - name: Unit test run: make utest race: @@ -42,4 +44,4 @@ jobs: with: go-version-file: 'go.mod' - name: Test race - run: go test -count=1 -parallel 1 -race ./... + run: go test -count=1 -parallel 1 -race -skip 'TestTracesForSanity/rdt_trace|TestPowerForSanity/power_efficiency' ./... diff --git a/Dockerfile b/Dockerfile index acb15be..a5465d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.24.2 AS build +FROM golang:1.24.6 AS build WORKDIR /app diff --git a/Makefile b/Makefile index e20b431..b186b9e 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,9 @@ SCALEOUT_PLUGIN=scale_out RMPOD_PLUGIN=rm_pod RDT_PLUGIN=rdt CPU_PLUGIN=cpu_scale +ENERGY_PLUGIN=energy GO_CILINT_CHECKERS=errcheck,goimports,gosec,gosimple,govet,ineffassign,nilerr,revive,staticcheck,unused -DOCKER_IMAGE_VERSION=0.3.0 +DOCKER_IMAGE_VERSION=0.4.0 api: hack/generate_code.sh @@ -32,7 +33,10 @@ build-plugin-rdt: build-plugin-cpu: CGO_ENABLED=0 go build -o bin/plugins/${CPU_PLUGIN} plugins/${CPU_PLUGIN}/cmd/${CPU_PLUGIN}.go -build-plugins: build-plugin-scaleout build-plugin-rmpod build-plugin-rdt build-plugin-cpu +build-plugin-energy: + CGO_ENABLED=0 go build -o bin/plugins/${ENERGY_PLUGIN} plugins/${ENERGY_PLUGIN}/cmd/${ENERGY_PLUGIN}.go + +build-plugins: build-plugin-scaleout build-plugin-rmpod build-plugin-rdt build-plugin-cpu build-plugin-energy controller-images: docker build -t planner:${DOCKER_IMAGE_VERSION} . --no-cache --pull @@ -42,6 +46,7 @@ plugin-images: docker build -t rmpod:${DOCKER_IMAGE_VERSION} -f plugins/rm_pod/Dockerfile . --no-cache --pull docker build -t rdt:${DOCKER_IMAGE_VERSION} -f plugins/rdt/Dockerfile . --no-cache --pull docker build -t cpuscale:${DOCKER_IMAGE_VERSION} -f plugins/cpu_scale/Dockerfile . --no-cache --pull + docker build -t energy:${DOCKER_IMAGE_VERSION} -f plugins/energy/Dockerfile . --no-cache --pull all-images: controller-images plugin-images @@ -57,7 +62,8 @@ prepare-build: go mod tidy utest: - go test -count=1 -parallel 1 -v ./... + # Skipping certain trace tests, as they cannot be run safely on public runners. + go test -count=1 -parallel 1 -v -skip 'TestTracesForSanity/rdt_trace|TestPowerForSanity/power_efficiency' ./... test: hack/run_test.sh diff --git a/README.md b/README.md index 9bed2e8..37ea5c6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ - # Intent Driven Orchestration Planner ![planner.png](planner.png) diff --git a/artefacts/deploy/manifest.yaml b/artefacts/deploy/manifest.yaml index 2fd0134..0c1b707 100644 --- a/artefacts/deploy/manifest.yaml +++ b/artefacts/deploy/manifest.yaml @@ -199,7 +199,7 @@ spec: serviceAccountName: planner-service-account containers: - name: planner - image: 127.0.0.1:5000/planner:0.3.0 + image: 127.0.0.1:5000/planner:0.4.0 ports: - containerPort: 33333 imagePullPolicy: Always diff --git a/artefacts/examples/default_profiles.yaml b/artefacts/examples/default_profiles.yaml index dfe3dca..429aac3 100644 --- a/artefacts/examples/default_profiles.yaml +++ b/artefacts/examples/default_profiles.yaml @@ -5,7 +5,7 @@ metadata: name: p50latency spec: type: "latency" - description: "Measures P50 latency in ms over a 30ms time window as reported by Linkerd service mesh." + description: "Measures P50 latency in ms over a 30s time window as reported by Linkerd service mesh." --- apiVersion: "ido.intel.com/v1alpha1" kind: KPIProfile @@ -13,7 +13,7 @@ metadata: name: p95latency spec: type: "latency" - description: "Measures P95 latency in ms over a 30ms time window as reported by Linkerd service mesh." + description: "Measures P95 latency in ms over a 30s time window as reported by Linkerd service mesh." --- apiVersion: "ido.intel.com/v1alpha1" kind: KPIProfile @@ -21,7 +21,7 @@ metadata: name: p99latency spec: type: "latency" - description: "Measures P99 latency in ms over a 30ms time window as reported by Linkerd service mesh." + description: "Measures P99 latency in ms over a 30s time window as reported by Linkerd service mesh." --- apiVersion: "ido.intel.com/v1alpha1" kind: KPIProfile @@ -29,11 +29,13 @@ metadata: name: throughput spec: type: "throughput" - description: "Measures requests per second aggregated over a 30ms time window as reported by Linkerd service mesh." + minimize: False + description: "Measures requests per second aggregated over a 30s time window as reported by Linkerd service mesh." --- apiVersion: "ido.intel.com/v1alpha1" kind: KPIProfile metadata: name: availability spec: - type: "availability" \ No newline at end of file + type: "availability" + minimize: False diff --git a/artefacts/examples/example_deployment.yaml b/artefacts/examples/example_deployment.yaml index f45bb8f..4f2e9cf 100644 --- a/artefacts/examples/example_deployment.yaml +++ b/artefacts/examples/example_deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: sample-function - image: testfunction/rust_function:0.1 + image: testfunction/rust_function:0.2 ports: - containerPort: 8080 env: diff --git a/artefacts/intents_crds_v1alpha1.yaml b/artefacts/intents_crds_v1alpha1.yaml index 332a31a..a4a8e02 100644 --- a/artefacts/intents_crds_v1alpha1.yaml +++ b/artefacts/intents_crds_v1alpha1.yaml @@ -20,7 +20,7 @@ spec: type: object properties: kind: - description: 'Kind of the owner.' + description: 'Kind of the owner (defaults to Deployment kind).' type: string enum: - Deployment @@ -33,11 +33,15 @@ spec: - name priority: type: number - description: "Priority for a set of PODs" + description: "Priority for a set of PODs (defaults to 0.01)." format: float minimum: 0.01 # prevents any div 0! maximum: 1.0 default: 0.01 + active: + type: boolean + description: "Indicates if the planner should actively managed this intent (defaults to true)." + default: true objectives: type: array description: "Objectives for a set of PODs." @@ -54,6 +58,12 @@ spec: measuredBy: type: string description: "Defines what kind of an objective this is. Also defines if the objective is an upper or lower bound objective." + tolerance: + type: number + description: "Indicates a tolerance as percentage in context of the specified target value for the objective (defaults to 0.0) - e.g. 0.1 & target 10ms ==> 11ms." + format: float + minimum: 0.0 + default: 0.0 required: - name - value @@ -108,13 +118,16 @@ spec: spec: type: object properties: - # TODO: add weight. query: type: string description: "This is an optional parameter - if defined, the user needs to provide a query string defining how to capture the objective's KPI. Optional parameters - in accordance with the provide documentation - can be detailed under props." description: type: string description: "Ideally includes a description on what is measured by the query - including e.g. information on units etc." + minimize: + type: boolean + description: "Indicates whether the planner should try to minimize this or not (defaults to true)." + default: true type: type: string description: "Defines the type of the KPI." diff --git a/cmd/main.go b/cmd/main.go index ca50dd5..c6dc8a8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,6 +2,8 @@ package main import ( "flag" + "io" + "os" "time" "github.com/intel/intent-driven-orchestration/pkg/controller" @@ -38,7 +40,32 @@ func main() { } cfg, err := common.ParseConfig(config) if err != nil { - klog.Fatalf("Error loading planner config: %s", err) + klog.Fatalf("Error loading planner config: %v", err) + } + + // set logFile + if cfg.Generic.LogFile != "" { + err := flag.Set("logtostderr", "false") + if err != nil { + klog.Fatalf("Error setting flag logtostderr: %v", err) + } + err = flag.Set("alsologtostderr", "true") + if err != nil { + klog.Fatalf("Error setting flag alsologtostderr: %v", err) + } + + logFile, err := os.OpenFile(cfg.Generic.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + klog.Fatalf("Failed to open log file: %v", err) + } + defer logFile.Close() + + multiWriter := io.MultiWriter(os.Stdout, logFile) + klog.SetOutput(multiWriter) + defer func() { + klog.Flush() + }() + klog.Infof("Successfuly added to klog output the log file: %s", cfg.Generic.LogFile) } // K8s genClient setup @@ -65,7 +92,7 @@ func main() { planner := astar.NewAPlanner(actuatorList, cfg) defer planner.Stop() - // This is main controller. + // This is the main controller. tracer := controller.NewMongoTracer(cfg.Generic.MongoEndpoint) c := controller.NewController(cfg, tracer, k8sClient, podInformerFactory.Core().V1().Pods()) c.SetPlanner(planner) diff --git a/docs/actuators.md b/docs/actuators.md index 5fffb80..17e9b85 100644 --- a/docs/actuators.md +++ b/docs/actuators.md @@ -68,6 +68,10 @@ functions in the next section to better understand how the system selects what t Furthermore, the implementation of ***NextState()*** can support the opportunistic planning capabilities, by adding new states, that although they do not satisfy the desired still at least move the system in the right direction. +Note that the parameters of the actions are defined by **interface{}**. Ideally a map is used to represent the +parameters. For example, they can be represented as a _map[string]int64_ or _map[string]string_. Other +types of values in the map will be cast to string to support the GRPC plugin mechanism. + #### Utility/Cost functions Utilities are used to steer the planner. Planners will deem an action to be favorable if the actuator returns a low diff --git a/docs/fig/intents_objectives_kpis.png b/docs/fig/intents_objectives_kpis.png index a8233ae51249f33fa6bd2282c4b8d9a617004299..fedfd6d2800e7e90eea35eb2c0219b590f6bce9f 100644 GIT binary patch literal 16959 zcmc(HbyStj+b-RmTaZRtLb@BIQ$o5G5RmTfkdkg`L{dWOQjk_!q@|JWI0%-+Gn*22;mM%Kd4!qLRp!kp67gYucPvxAcW8=HfziJh~{3tLt*`xmZ5UnpQ; z;1Hi{>Nx-TJq#=u$34?RUCI7CJ4VZvHqLb6L!z#}1k*hehJ<>(!vG|knI>D#daZ~9 zb4ER$qqf)^6g+2tZe;NACq$@4~xyLklY$EBK#ITPS3Ta~f8UydHW=X)JWvHF(9I zPqkD(S! zSSv%Mzm$kh(a%ZQw;OZ$zs=K--Uz!dg^Q`YnNa&PDsff(zO*(x1WWJa`N!ZFDJ~AF z!&pnU>#bKQ`M9X=zUVKz5_ubxZQs-0HXC+iyA`o?Fpzv+mOI)o)kA9hOdz8H`P@gV zyu>)LTR(EuN`mA+bBo=4G^c3OoV{TFg>cDw&1Xa6iO(3EZWeR&G+v2siI{y4p z<6U9UR77?@kjA;&d^q$*I;!gl{6MFN~S6*(W1fPm-5*Fs`e{akX> zy|otSjjsnk`?cRal@>Z`J}QY7xnN*r#fm;VbFVs=nq7@6=!&FtS^fEQUtpa_98(n; zw;NA&6n6RQ{BWgAYv*Di2wSX8Om(7MH=oSo=a(eCdfUE%0a|79xT?c*e{ms)g%3_^ zn$VdLpTqxbao=BxBxKf%k9fJ#i~B~l$#n-VL#6u#7F8C7?-^`2$?MlHaQ1E2JSFU} z(*+2z@y1|WaKu?)-_o$psdyYLH9T?P)!zO zgt=-qo7pXcd)XVK!ow-aJqVX(`>OagADl80&-JxhQuz1#HQ8pLUx|AsB}!!4;ot35=5Ucx#u%%T zamC=x>NA+RRaGN!W(aCdC)?8=hpVH0QhelN;t1k!WdHkrS7RB5E7$m*s$|6ZW z@%}~8C1Yt}QL6BY^6>)IM90G1a@4h1q0LbW7S>>wC>A=pM#_EhxOU0%pr1{zmO{Er z^43#JVBwKU{(Qp`Ha7_8PB2yE@dvUvw0r+)LVubQhZ3@0+lN>;Plz{vGG57Mc#B!Kz0Dc@g2EWB_xq8?#P4Mx$1?9L(i)lZ zU`l_Mit%v67vYKRWmiJ}Av1#V$1_P3Bz@W`?<9Xl7@0WHJi4j~G7f$xi0i~D%*)0m zNMAXBkQ1IL@n&$!muFFV;&^4?gWxbRDe2Gg>JTh#wh@6J0Tl4mqc?DYg{a(IrE7$=U;9Gf3l2g0Vt} zW@(Nz);gl(2&mP+D_`${*rD_c>Q<+2XksR9K&xT_aU4i z0^+)t%vjw%8r@d+Y02&7F)YOyKUSB6RzhZnp|R@Z(zA~)ZS zhFgMum|rxoJ0kcXJkR$v?KQlg{^5FiWL&gMjjddGNJLs?SYjKywMk$j(Wt~}DfwG% zzHf@LZcQ2VWPtV~pa_aROrB|+gZCde>k=0>|zurem)SYJK1ZlhxY zr#XHP0lDo`vfNNBZ`qxFSLM=%eQJEwxZ~dJ`cElf2BNl9|x7vkeS`OQ~rg2rORwZ zRzSP;>5rUSBPFc|J4oxpqtVq_?$_;F7$L!_qEoLZp6r{=*v*o#_p>tVR*&n)_dU2jcc*&|E%3;oirF6M8puvpYm1;LqM&s<<*UBxW;Sa;u1+!4%_L`yQ>?z69rs%7Om`J$Pp%W8zZ&h5~$B{Z6`WPU2|i2A`1s)I2t__nOfC zf$F#_a=bsB3l6ToKsW*cIC}pBgD!WnY8M6`9yDMziy>I#{EkvPUrFJWJE&Rc-YmG| zpt3iQbR;}bC^PWv%I^HBzP59HI-kSuRP<#7z0;Js{aQunaOw<^EKJc+W?kTDJf3s@ zGnlQUE|hYcqnUjJ%L{FML#sl+v=W6W!So`qDl%C4#nRGJi9vXmkL+oBHQXx}f;Z?( zHqpBrtVmNrUMIFWB0igAr5bOfB0y;Jy*Xdwnwe|yvY9BKE>s|1O5-q-883UXHSw;= zcKXptLbiyHXPx!9^?2Fwj7{Z_7O#~1{n>#rSBqg3L+RX6c1*L2^eWl56Xo|vsf2~e zYTULa9!``7vK7Tvd7L0Jq|aEee4LCXbTH4uF~Biu=p^qlTzRM05UOhp{&RFZ$gpqK zZE}4I<=OAASn_r!^6X)#F{#CFs(Y7 zsk5;f&Q`jPTyapHBNKET;d`s~jzqwj;RgZOT10uZN<-l~ZTuEz)kSi=wHiOsxrW;?SO4(E_-Qg3+v`)4r*&aLy;Y&T`+IV$*JG z2G5I`_5gTN9@}2S_xJAI)wW6jwSnI>5?{oJpP@?3XO;-Gfz^9+ODskWMAtq)s#4wYA#JwUD zOy;AyaT*pz9x5qen2?L!+jHn8U*UZ8$nk4|T=zX9A|y1dIt1mI%MJy$miIty*p2tl z>#5~1rEuQ-Tv-tt$Nj)^|KAI!BBmb@SbU`S!SQED5Hg#=M=7_`V0am3O=Ob+iPK~@ zqwFMw$jyted%V@!;3OSa9)DDDJ&V4_8H!1Y^l70ZD8l*~4GW#g+E5mn_I)BqVFE72 z>#c0BUsTHBe)~((4{j&>KDunF({ls{1@*Dz@!HM$XCS{sLPRVqEX2*|+%k4{=8C?2 zeI?{~{&SbP5k(^imm^xmZD*#Zc!TgiWwjbi=dLtplG#qASNY;xB^ib#MGTW#G}^-6 z+T?wHV6=Y|+;+q0b9wS$vW;3)h=+%W7SHn4lQPYK#bQt1u&PuR5&@kWb2*4>0IZ)0 zflMZjsNm(Py~WDv>Oqh`U-vQn1NqmK%6vR*Y_U@Wt>j{-HsfW8yh%XG@ni6dkkY3< zI9W}x=+$qlh+a$K?-Mm}o5}vI4VopYi@>5$QBmb@hMxdS4v0G3TkP5v6`Mkb=rq{< z2u8&RiL5mWr_x7aIyd{4W1=%EZpe}FK`%}BaJEE}roR4tMHhU0d~Pgc_O}GNGz=NL)0sxWi(FSSAj|dk98c84@OmDm{nsjIw3|Vrb0ORPvdAiFX*_vr8G)il3d(3bv`+3HZ1UWGnCYK0(}kjuW8WOPRpDSfC{-yWNulljtp zz7)kKBP|`^)D?<3n88c0452k_^EGfe^V)4eJ^>b`0E7<-4z`_b#9i%Qp`h#5MVXz5 zCVTnYd#&B9bFk;4V_|7|9n;a#h2H3_5;gRUjfg`cJ7F+sn`C3j&0*fIe4ej=p?7_8 ze0jR7avP?AAJD-v@Jqbu`}^ncU4~1F^74VU{`=+cgxpD9MY zMeKFJx3p)K>QtLj22zInhXkI-skyn7kcXXIm=&(6l;8YpE>Pph`TCDGzKsjgHa;v0 z#qxZ`8UgvozAI=}lGg&yR{!FAGONKy&*RO-#l<7P&%SU-vIb4AN-8SayKxKksUq?^ zT3X4mhZ?mw6BuJqZ)N;m@BK41hB2!gADJr>HsW`NEgzrOS&v+7O;+Xzy4ivHavGlS zR}1ClDRQR-Rrr!U*?T@)4ksrkP#@2=`dlV4YX=T?GYL8T4B|R_B+88rav_7D@1P{M zIyg8GIbXpq+Xdqa9}aM`6T)z_JSiI+$&;jl^UB?8p}eJHAqZqB3F;1yh}Zyi8aT7X z1mCm8u*;Jj^lUgacns)hf*uF7O?}%}@ZAC-Co8N6*91aUPNU zni*0EilDzDvarX8$Itrb&=2tM_9qF8r0Q8KY_V|w?mWO6-cI<72D~%{LFc8=S3C~w>ggC=xSa^?$TSVRa8fBk407QJn@);VU&f?lgSF8+Qe4h0_=|gH8|*2_AV>%7an#!|uAk#WB%C{9#8mY$ zB3JWkk;?m3+&<8%oJa)b%f^OwQ-#?~R#+sOnncMVg{mftkt3;}VJ*PiIht8?gpeor zF|t;eIw?8h;){tbQN(Pa9W>^WE{234Q;C$c`E)lrdO%!h#QxSntpI~}O~Z@t^+%$q zMt;|IENh*nGp{7PWqTeDmE0q|^9hD1!SYJxu|p(PtfF25U`z+Yk391rJ(x1r4eohQ zY}b~r&UasxQ|J#5B0FU^6*ZVBYem@huAfaXOBLCqkv0+4gkvY<`)&yV7V;QIEDd_d zdf?dw?g`|?^l)WVx5s|#pkG;*SDb9Uc%<$2#w$$`CF(ibtgmcHPtvV@p-c=(DUr|O zOR>Q{z8ckV*6>OSli2E^k5@I{MwNQM&wg;pP5JlrxPlmyP`g zZdByL&8$Ovpeq;mzDDticA5B@lJ)9GCa*m?#?wUtg%sAGiPT_1S#v~9*wLlsO`%2om6_G=dS{SRKJ6`$~rnS$n5atcqGUp=>za_c4z zG8F!_;Bwv~U1^&Bs=h=G?Ojkihzn}~o8cv9(P4Ml)RaDq@Jz~^c`dB4D|#kDg(B|F zf$%u?r6PyL)8)LuuE_zA#fUHWX0Zqg3i9!#-p00V{&z2+#+eeqnG8B_jy>;*B$U5d z&TR2Ke&ah0M3Ssy^h%!dgPIE3X6{VL3Qc($NbPv6#~A48>AAR^fjM_?oJ4qPA@9z$ zDDYYgf4#gjKyfn7ra13lZg?iGV&yF9C?Zgfu=XwAJvp4oFZuVA)vvKgnjv+j>Krcu z$}KI2voV(&`JGl(W53gH0xiASD9!d=fL zH*C$h@hK>bD)c`9_oe}Tx37sAOxv>CdRcGxL;kwPYU4VrTor|SfafBMj=!DxQJf9CbL^&iNjoh4T5sVoD3DaF(ONL8?uXg__=bhEfZ5XSo z<5kzz)O<-{Q@P#_GS(cDPiB!bB0&r?aDEEOAz3-cc#{N9(Ps1wVoW*PKb?UfSwYmY zm8E4eqeke(ZDhBtR*7oBCgr;2=o{%LxSAQ|exIW60mlw+wztsHpZqZ2*XQ4JFWg*T zM);f``~*>N^OhN}cDhK3)YVXtA}YSe>bknRntiU>VNoK9=}AIjVyThQzqfD-Yw|;}2!8`BHuC9&y zxw>+Q;6Z9JSUT(t#^%1?Tn)Pw_9TAlM#+g?7~ zox6XBRc3I5fRv3%%A@J!BCVq+6+aeD&vpfwm1~yJVL(u0c^kyY$cRs;m@4A)(KD<9 zB`Sg;A$`99-W~5VOK^w;PEU;h3jPWdNG>jZ>$V)VCz zQY8Hsl{(dLe6KHnwpz43N>58W->w}_1__gHtz{S%8MU&sBP*eI9!@Z2ay`rj2dw&l z1ZDX>tz|AlBI;1vX&>#5h6t|Y<1o_2-;sH$GP-sD6Dg=?hI)I$qoWIUjlK*O-iZwG zK~`D{lff#1lKmPS4)gwBO=zu3-EW>w+^+N`Iuk4L(9#AgB~uEfph2VPpGMrZnfs|< zqC4(k(=O8)Pdv5JgeWs})Tcb($G{1MU7*%DmEsVgi4%;8*P-{x>TqgC6JQ$>B|K~z zrhDUFRo-S@tN+2#nb9_KTb~KKF^DdzI}!)D6Buum?2Mq#RpRkA^k2N?F#)=vU)cMH zv{%JJ_3o<^r=|jgKxDS7MjzeB_1mrT3$v_Sl{1;#HWL7-rn&(|MLMs&a^{~>+-^`_ z4!kQ<(Lj32m8c@3Tc%kGEEgoculX$2)bH|@ zn{VR%))Y~@Jph&a(ny!K-L zyJjqMVNZgBMDHJrzy0-9>~L3@@pZs=QzVu@#o2XMJ?nkDH>zcb9H5*f@J^>14g(^3 zvN;~bC+v+fLD3r-@uyE~9uc3WsqB>13lRX`>=Ou;0nb81L+$2UvwlAdRamTdzn978 zta@@cZXQ+wHajRPQRsR!1@49fkHMIjm~0nokiifjqZOV|KNc}u&jot|gx=Q|NbS;VvUppPwlqP?Xm}-zLP_q6~z< zZ&I2ff~Pq_lc^kLlX3~audmPN;)qeZLLWpxWNdQp6J6x0eBQ09sNKdx%qrGV-u48 z=a>;wQ&V<7J|?jlwc1YCfZdW)R;Ke^L`L_x+xMMZ6jPZ0!@~kgNJagRvoNLm|20TN zjB}i?&sdLYoU*bsIYf?@zo zf$-7o0b(Z>nZWCZdPLNIpk(Pj2MJGS00>r|3jxR{zkVouqerp{s&-iH)Oz=nG2sXw z50BjU%p~=y;%QyvWJhyzb3s7?@cgZZLxSDkO4RZ}(iHge*X#!X5aQ9B`at#E#Ki%$ z-YCUD;t7uM7$7m=_rpH=9RD}*D4?JOI9i+6shzp`N`YKrIR1M}5X}K3Md7^%i^A}- zlL~z7@9PUFAmO!Byb}kO!+y3g@9|d(4%2S1PbS^SNL~aA@zm0wcPZ%_0~{G>iy8x^ zG+*kD=#J2#fZC@HS#)eHGW=_l?e1==O?nvz2lj0^c=NX=U7_7_eM3V<98GS!bKnms zYyr?SisH;>^DU7n;m9QQ{JVWy0BpX{34>|eJ|&9LXiPd4m1FGg}PmuNDh@rQ3YF91v24k+&Lf~xmls==K~4jy9G&@MPl^@O;5uc60n!@i^w6j^(-UVa z2-sG{ngD25Pc_&ZrMlEV|F#NXrtS|G{mFDn>Fp&i7CS?@Rs-OXpVr%=YCL`8(=B)N zQ~iUZg(^ed8`N)YK9{Z_J)-i#(K=y}04HmFgHGLhgLB@6E!tiIDkC-Q-?~*jdKg5G zM@ss=T$i(sxS9uQ6nu)v8X6h^TE^hCIL7Lpfx77>cCz5-E-9=AAlDlg#Id;(u*(^z zni2ZOA718v;?@6O9fhj8hQ`+TTUlPU-;NU^>*(kRRyRGa^4B+@eXg9z2byC#m_S+i zTtR8(2Y!4DzJUOyfGIaHq1j6A=}a;f#rY=XCBaS>T}9--+p{gx$PvZTJJiljl9D z<{be?OF)LtZQKs?BqSGWCc8dXw2PMq2_9~&`6Gv|#0QuOag>rEtb(6rl4r@EjK%!Z zwRuC8#K<(Cpls&uh?8+K^JybDXj2I7-pFR#jW!Rq#Qb^(0Kt5j(;t)NFJh3xUb|ly zO!fCP=!%g!Qxbf@~ zulqPYQuuVMK`|OT(@I&-1+2?RKrU7~;r;v|(A5ds*z585B?}P2vyM~l+GYEYKc~VV zyUOACep4xpK5id)I#Irw&@Q9Lw;5vXXJgfLOPP-)fMe#2qg-$hZQyqh^TMi+Rl+bk zA@k+wGJM;UjQ0vIuVwCIOWuZeP4jz%vHqYd#ii#~L>(IGIFVd)etxfvWb21^KMh&1 zs_2T}Aw|(S4R?r5eCIJf(UZVdCi>hMwk|tyarZR%QY*;L`wOg# zMNf9DiQah35L*Q|QNH$Myim4$Wu4-hiM&P;t}t|UV*7^q_IOy=C_AbpD=j4S{P&(GE=2jWfKQ{zM*Bj^#n z81LSHayyY-RCG3>5*#mS^^5=I!;l|(7lmYP4=gmZ&j_WwOm_CqUYB=|Z5Q6QXgcb1 z#pI7ts3MbvcL?x-2C0VV?OUhXrpFA#t?&XlP!a1yue5l)Wq~F;+9+gP#_c^gY+PG>r~x*Hr>SAI^eU_{MBwovO6p|I6Z?`z__Du;MS>=;dZhw58I#k?F@3m*D z-h(Rm9ASK~L#5-MfKG#Q)>O|o_kcrHWVRN;7RwRg-$xUTvAB^W+bWPH#tUtf-k!M8 zJ{>OfTVEDVYzr8YwB5OZo*c%vj$$qHo5?jR<1yDG>qoN#(+0t_%|fYj6|-(3zn(xk zzJ|kV(563K{h==q1xrICsGanhN!OIN(CV88%n?HylGVELv*k}D%N+=o0*ds#wo}|T zk=8=%3Zf=7wHYJ$#?*?xQnFV4ZV&DK)=8w^axF5Ora*|Z5?-cE&0Iqb22vEHQGX-9 zo%hx+RC(an((+qZ54^D8mFOl-hJ=QBbBX9eJjeQPV0dsE9N#Ke-O`&_iCs~RL9TP~6B};OB;iL?A+g@q{YpGaqKIH4uiM%T z*4L1iQrZkXv@>Nyt%1{UU!HD+F3cpGzvL0}e!JkstGgkPnV*lV~n*6&lVs zor%m?T^Q1hQKDZG`JmqUcu1Q#;cKmZp@lsk!FS_Lzw>9_H4bBKZkcUq=`Qq~P~M2i zc6-rA@d5^!bIjaanFsR@x25@F;o9dXFVq)&@Iq{!Rf4@mmRLhXZl7O$cWwLLN<_$0 zm><9MjlJpmecj)a13v{%k@+UbXyV?P*7qXAHY4w&ozgne2KIw%Q^={owDm6}``MIT>n(0qRSBH(-1^Wy%SG7)r4bnjLb@H#1yU*GF;XWfSN!yNZ8&_4 zDNZhF)hb)gY}*1b)9XuT3bNwcP%mhp`wJbG#)VzfmTmv2DM>(>l;A)tD^v+ou1}ge z=F7NW7y*BH)*{gThHHcWwf%DpB!m(V7qck@H-Ys%sF6u{-W+p@BGJI#1xUEf+`FKx z($E@oBVr-ci|cyCK_phbMy{SoeBl$=(Zl=Tm<&o=2jjk$EgC{dtVJ?5PyiVli_5r5EU!%%-g~_0_-lbQHR~f8b+El=lo?y~xAv6uO09Q*MnKt_ z+4O=qhBUQhA!@g1CH#1iG$}441N47Zm_XmCT=x((AK$d-kN=n+-1t`9b*u{r0h5Ma zJD_vI)d?65RPmq##$z)PEray++qbQ;QYS!Fn0{~siX|d8!)DD=b!?$M7}_^0#WGYG zwELh{7N_J$31QYMF9C%X04%9OhD9!;i}=kxMVCE!3xFTD=^BfgLz1BKRpE|6ez%MV z4Y=E2fnfwJaX`4LXQOXGQB_n}m<7;1K=v!|t{PomP3N`&?QN{D$#HRj=s@AGM1yhf z5iK|OG060DwFF(ab3cQ!%o=b~ofDC1F$&n}>8$|#10ZucWgd9jrEOOBkFGnOs{?7E zZc`CCNBBzR+dGBM*6aWX1hrPf^|sSrXPGs$4((oEqfXn7m5la85qI!`N*%Blp5HH- z1566hvxT7j{r&E=%uEb=005Z+j)S_odU!1kij(poyvz4u6^h2LBnCCOgTwQ+oWzzg z+x}naouoRJmaDbHB7l^k+n7Wy;JgkXzZx&5ts{Kz{lRUSaMryTd6=|D*Bz4T{QUgq zr-h-`_@TB9_VY!)Gk`6zzx+9(1t8(zM85HXE{}-A!l?@57ym7DP$DtWXTb8jbR~v3 zlOZR`Y5Ow5Y0eY25^!BTrs{}={cX%AB$aK_*U-j ze451J%D>CcXJ~WO1N?({Tw9mBZP>i@aX)zHy=z@}X39_^R}D1W;k}k)cysB^&CSJz zJiW; zq_D1M1g!87LcBkP2lUqFHpR3@>@}1;`h~XYpJ?8Cs2Dq81r>pU)_;G>EOuw6E=tCK zF)v}>m3@_ZXko$l(Lo^YO8|o3^38b#^Al^)jk$tiDSS|7q|pY=-WWiHw%$FLUS+TL zdWc`&uO{}+$1Br%Ctry0Yp1+>gwH{3R4FyEIzdX!fSl8E0K#JnfdnCA8`j(2Yf?Z7 zw*)*`Q0#&FTt$e9tbnJ9@v!T4+$N8r!Pp!ij^If{>+Yp8j@x=> z=}IF-d?xci2&fk*Ug=H%&6#a`klimYqEkRY>(u@pEF`kHMvzXrFTF@)8yYBE5)Cszc|{E6vkO>nD>Q;KdJ@*L^o=u z_~)}$O6V~ws*&>AH3P~ubk!f7<)#?O=eRfMP<#OUxJhCECcBLI@8dlK6aYh?<0lX< zaczKO3h%_P_z`<}cu)p{K0yA>&;FFQ;|V>`Rg-)C*!^&jm%?KOcWJE-DHPRVp*No9 zi7$C7<_ka%_;tm8oF9}9_&%8eE@(q<93s%!_+6dAAOOzyR}3Vnx0!r3oDmOQMJ`W; zQJXR^V38n_UodAR7_AT+dE=(r-&~!uwH)ESd_}^&35W~qmY_G`uLl>i{jf9h+m~|P zTDohCdvqT30QRLWI$Hr+uP&)dUGJc{$cSu2&SW|uGx_6(x{q~zrn=APkPC`=09ev2 zDP%gW4TWd`6iI6li@?yaY_L?LxYObrXr1@=J~mnnZOvCZw7bXkOf~J1#ri5}W5HMi z3lbf-uLdHb=f1f(*-_x8=HS5P>43Iq8dFAkp+@-59D-4yTZ2NqL^1Myc`9v-Fkd2JuKolCbX>j9;|?q$DJ0k1(lG;6Z!L-WYo9mQSC~{2Q8=S=cQ_YA2qLb%}w% zgVB#ZRu~r{aef0q;;kD*LRA$#lQXC#R&8CFF7Nno&ci*l_28 zRAuJnE5=jJ)-&I7n{LRN4B?we7sS)sKN~ZUPTas`Fu09E}v50f=pYmj#d`rMk&4{Jb!K2W=L^L&^{kEs(e| z(9i-G7Xfskv{Gn~}kC-eS-EqOKLf4y@msfXyiiL%Rjjf`n_}+Ku8tt#>{BpoC zTS~uN1-PYBfTg{dGB?`B6gyKux=R+y&_izg3ctg<=nrYQvZfgsU1=T)z%6%x1_3T& zaDYi7oe~j?j{mZF>zRx*ad+ijOd0x3A|9DW@`DG5TTwaIhHPR80JNtdAV4^n1Ozxh zps=|WS@(bWF-&?P%3jJ%{0eQpgrHLEOE3zQn$QdIGC&i2n1shpO+#ZdR-y(xGXTs- zWlsR6TC9|TbXfkiPyw`7yNNoQy0`K69zHP$r5= z=Ox4G05lkJjvUX;QgFRNt4=%B$u*PJhFCOjaziN7daq}4?I=6~M6*5D!T}IJLBE-k zoBK)K7$*d{W6+KcG@$ebsAsq6fAnLsKNW}WNo6~GPOInED1N!B2;4I0-wbZPDSyhV zw{9Lk-nN^ULL!bV{-H0`=y5aTRW`OPC0%g8$xOR^kez68Nu`-|=O`iBM< zD_fOqmW8we0_jtL2ZxD?`G5zqlH^4xB9h&@*-y?O6d#tA)X9p-)L$^L%hWhmSFRkq zWr?1}Qj^Y`(%GW2WJ;OAo*af+XZ>}7iLcdFkbZ$TBqwE*FUx_IS}|wUs4hiT)q9-B zR1o-W_a=2qsBzD@ZWpJt1B;;Mxv$ge?P1z=KX{0_!}uF?ZDS)cjRYkuf9z_F0B%Lb zL+2Xc<}N_)z#KD$EqsKdi=)S#Z1s84ZVjPKX?o3qOPM}1s`sn*xy4@3={kG&m)|?9 zs3ottrTr~wc_CC$JE}}I4{lt1M6faO+ZmJOcQ&c|-_vaB*9fRdZCUnb66g-7Qn$-0yLbxBO)TgWU>+o zUK1%ZZVco`CU4}g04lqk8SZN&F~PV&*OXa|@rP!V3H5GDHQLpW^ijX{*rFNW*VC@4 z*Ek0&dL2059`9o)v+UgYA)LkMiNK??B`;@)sm1RjqJwcng6H5f2k^M<{L=bdj|IFeWK0q%QBvxy*UlPd!q$`_Jc3&CBw_BV!F_ zK_yYf25yC1864h7raHyE{HR!E?gTf@>p;-=mmXL}<-!_yZ!bZs3?Ff_*NeO9rxaSf zALw&Z_BH00qjpoiQOb1wE@C`t9e?etSpeiz<+c)3}NmV)vb)2U2qF=7+6Q3=SPCmyG?cgnIfaH?dsVwpY zhOH;PqCYb{v^4paxyAi*G0I-QSo{h^F9Ki83y>oMnqhp4UW47vbZv2zxa;eLO15pd zOxDr#%SdlwpY%q>;&$LV&H?@JNr`G6=t79zUUBaCYhAoPx7fIZk4bOfyXpV%J+!Eo zdXG_6vN8f4#78Y0(hgoG@kXR;z^t38f02=uhUb}o^va2J7rPY%iyV;Ji`z*JJ~~fV zn;};xU5)nxR)jq7k51zGM&+P0)K)dr8t!#*3v#dR>Q6DG^NlZq+;w*1$O^v_PZ!w= z`!pON;2MW2*rwKlJo(kszxJ?!zu4>?#Hc{Wc;baz1LN62w)Kbi&(KLF2=CUq5TlgA zMKcAXYa7jWbe7UkyDWIuIhkA_uPX{2|4*rhkoC^56JKq~@WLGg`QakU#D0@d)c0Ok@s@LVX9xx3#R*se_;2nc)x)P4Dme@=ZEina&Hd6y1`d_+Y; zxs%;&(4$Q$+{HHMw)Y9b6A`-#58z@c?>x%?AJ9 zX%@gz?%qneGBP&tb-2E8X^8ibq`A{0G0gFrOThULFrch7|K@bIl>aY4g@uEIg@ z4OsF}Y*XuK*z#b)~Roz&i=t7jlpL6#vlSVb=rQWaWC8 zx`9}Mk^uHKRkP92(e6+xQH=Ky5AVNZ17OVdRJE}`3uyFv0gjt-EQG@pz>bmLcneHt z|6Z0vhv|#Cxbu*(S|09&gWP45z<;f0OdKc0{CDqV7 z-K3#=nq%fPzv#2w;pjTeWiXWH0RtRdIY~6Ak@JU*Swi*Re;EI;8^9KKmc%x|AFc`d zxS;$r)kpy}3sPRhRx)-h=2**&U)x%^+yZ|7A z+RQoN_yuH>V;b8oe+f?Jh^}@T5g#5UH6+3WfgB;ima4V=uU8&YtAoN+R8{_$@!!*f z*09-s`{{2Z$0s6^5gmL5K^o)tsULOSlfY5^jnM-13CfuAhoFz~U#i!)K-y8o#X11O ziz5#AN-SFv1^EDA=6DVW2lplT;$;82AOGj&UeKHV>)!nNZBGjaZJ1-R4){Mgz{p7} KOI1i12mTKy0)6uU literal 17366 zcmdUXbySpV`>vuAl8S)RAt2Hr-5~-B1|^IjC7na3bcY}%Ga#VSokL6K0Ma2y4&626 zdC=|m?cZ5vt@H0Wdo9;?X5ROS`?>SF?#JNg%JTR($!}h{as^-EiR{ZOSFZMhKlIob z;2V|a$!TMuXJzkTX~AJ& zZF!4Jh!Tu&VXCTW|M&YVSHUpOFr$~xtj8sZY73gZ?)EF)9wL1na+@am!F|)FK%blQ zb>&zkJ+8xVdVi#4U({B&-ns8yC{|)URpU}}_(-WMe~_w3E-`%_d#e|h`}PRU_b(D$ z24ylA5^rh-ZRKefLxe-dtA#kc(%r6UwSzoq!5<#paF%ylMYjAI$E>Up#f#gb%-*C6AbaOd0) zQA=`T=gy(AcbcdyZcdomn#wsft8U~lWr#xD-8zvhDt@l}8khs>C)q6~%SlF9$9eI( z7OZV_(IQ!;Ph+jLuBa|*Y#n-W_7)euOSf0r8%X`h{Jca>)z42?(ddpX22U^hutUsg zkBP#{^_K?rG9wWa>rOLF#o42F^S*-LHcXS}Emo;)$x9|e=aM*NU%c{+{^*=;w9d9j zsB@Y)SpGde9S5h-3u|+;c53lQuzy_hDjLtx^HnF&-kg**SCG60Yao~Hbzkwl&1jV- zLuahHjD?$x=(5V=sst%ZSPzjM3G4}#n~yp7wpF8(e6Hg3PMP#4QRPS+=QMb6fTFc; zLQRj#`+hFh=Xgq`IyY;2S{SUp46L#8#JHspYZfG@Yjek_3chx>y3c(OAJxP$th7xy ze?S0LY+`D@az*Nmf~=IPv(7pk#~EMaqRlsmpI%CT+SIbF<)uzYbcLEyW?kHP)#sPy zrZ2RWwDUA-zh(}OyXNI>Wn^Qg6@AotSWa7Hjq@W4J2-duqAEjdGl7>Eakw~@ByxBz z-s`f{Ee2iehC?^bCa1e$LfcW;KD5~KGpXIQ*o_O7&f94`tJcNE#UEc+X=>>p)8!)% zHm5mzJG#3ktDOo&sof8MI_*$q?rzOA?NCz9yH%JpL1vzPjENzprml0@ zU9=$PHAYUB822LKB4liuHfP5U@HjpiI}WKz!O1)K?%ivPq^Y-C=-l{O-3t@)rL2Jo zSWj8kpE(|^z3#YQ=8HwB&wi?}j9yw=q7reGef;|zL~Iyy6hSl^=~=Nj#y?7iuS=Jcoj zmKE011iL#=)qgGZr%PO%mWayNOH;KYuFuTcypnVQOAaWpnyf5xN82qeFZLwil5sRo zH#f`5Ms&ro`)Y67wSMYd_`R|IBq7w7U)VL3%Fd zuj{ctG*#kwh|LlgV#P_VqoFbD3|yCF`?OvJXWezcXyz z!NcvPe=^8PxY=M56){Yrqq!Nc>#5gua=6*Hm`UXQFqHn5x4shuK08JC$}eu`<^q^j zf73aJh;NuftMo}(He~u?b=Z5FfL&B?7w^T{rq^7B9s=j|T{}ESH;$vewk6eg< zWvd@i;)!U9x>iqlN?{^bFB%jc?oWElP2bztt`LWiq9|;@MzviQ$40TvZb30$*>-cM z=Uz7QMv$aRig@^dm!rnBT*c1yyke#*r%mdcHtWSt*MdUTsL?ALnb(BkX(5zUD0)%9 zN16o454UMvrRKKhc8QS!9Ip5@AS+E6nA%W=O#G7?YpLcGP3 zhV8kwncp*RZ*Na~U5NF5csde~o?1YLhlXlbJF@fFyB+3#QCG{ijurR-skUo+7!ph} z`?e>{rQ`DGmjuahv4_?3w91Toso7|R?Xhoo9xpS`|H6Aye;Q28T=+hoh%%IsPO1g} zYfw4G&qYdnWL*m*u^tMz9K4-|s8-sm6QU6aQzE)PBY61+QH1aR?ZY5EDLHv6 z&+ADjhsRH@vdwX~zRyxS#zi_+M+=nyf` z0Bko&q?!uQING(VLKl&9>5SMpHZZ6Y{ES!`QhB%n|G1~!5m`6tnA*@+-`Vc;#%-K>KJur{K_fiS^xjy% zEz30x*e{Y#y1_^9*c8DuCZk35{~jw7I@9MlEk{CLzK>8jxSwHkZr)F05lb(7W~jvG zH-S!K82WY^!gKhN*Ol!&=L#sV#O!m_x%&>k?*3eVrS&1isLSq+xR~8Z z(_U^oih7nY&Lhh3dm(6+h#z z^?@$<+HS(a9E@VTB6N&Y6pjf_qalhr(Lf}hZLAg)s^_AqsjA~9AMKg^R5`GJZ8P5< zdw-rbg{05ciaN%Ulg|ZM#e|+Nid^7wAv2TVZPqQ!5)b)>VW>ME)*@*?Ql*c1DA*W{Vk%Cp8T-)Qlrf6F~@!F zv2t)`uQF!Z_o+!>?4*Az6iX<+km|Rpmti2FspGk8unviq2jq^E0VVB+6uX+uHXCQ9 zPyMPMi68Hm$$8!X`mV129FLXBDIe*)-dmm=Sb$X^itd)LKcDNm#^5Q~vqNIbQIOYs zBJvicv3TJy?l*k}c6T4*{?v<;dLGi~_}iRDZq6eu1<$q<>NjvNGT7sUJ)n2d;q{}N zdUjTW`-XL9v)`aWK4Ux;U%+M@t7F~NlS^S;aT?TB-{VfaoG3dz&F1;xwS~Y!EpGg_ z#=Hi16sBigZ`1r_UpgahF;`i^^uU%O?pMA^kS*XeqTZNoIV(frA{nSJ0#WdIcg|rjV8B5hoW* zUBSyMlJRqG+C?qfO)nT6|4qAmS9+9|_bm0e3bS2(pyS#&@d&9E6H5X6QrO5aqWtsS zTy`gp+xps1T;?PDM>f9|RXYm*&vzRUKsW1RPnRNd6uzg*6NNjTFShIHR+!ef%b>JY`peesZbuW8%z(TLi@TkEDU8 ztN2N@iHv!xBk;6k-2Vtur3e9~;J2s*AjSw}QuCxqAJ$CqGtnVi)kx|=9^ zHQ3!E!;43<1vZoxp$j*OlbxCL5Au~l58wC-Y>Ia$l+Dq5CCKS191>n#XJ~^PyVyN4D3+AotM7hv=a@Iy%}5w92w0BYUB} zVuvT2^^teE8@sx?rl)EAeSLiw_PbLg>R+Tkwve&eUm242zX{UK*BPD`FY~k>sp?eO zFLN8TZdtOoFd_HJi}Jo3yID`G&}?}mCl8ld(u0lpz;Ys#adY2Pllu6k{^D5>Rxq0= zU(YG1bKRU08!ZQUA6(c$A8o@S>%P>N8Wt9I1DEvZ_?YSb{U<8I6>f(c&u2h}^b;+ge-G zPCH_lU(m9!EH`*x^U^To=C|DlE%8!u`MBTop^~1VJlK4f6~rlR5LM(eZj^)Uay=j* zAOz`;FD5EVq*WY`Lc*K6Dk}v$<|h?mn6{(w$@w%(O+MU*F6^(4ytwQ2`s!$*W|97L zIy0Gbe_XPoay)jF4@evtl3Oq+!>>3FWvfLyp2c&#@qyaP5^YS!z7Bo{TZR2SM?`r; zx#ff`>ha^puCA^-@rj9veU$pzU^bAk@2LH(ayVH^kG!AAdbsgZ`?I9?LmtIAw&2>e zu@Wp1A2D<}9u+DeF!1E$gdyQ)ty@J(imsVi_Lnyj>Xwt0+z%dPn)Ii!Yn6@`8{rlV zy!;&<<{=JtmRHqK#e(r|(cIkJJAzJ~A6aS)(;tW5vE`V3Ew~M`{u(^r7Q~vX5A0pf zCsQFZZi}yqvHTXJTyGoRM@7}2D^#kV!Q>;U*)N^@E(8oK#R019CO-d_G@xYog421RSoy zx2|G6J-r~cqA=HUbgvgZO}~|e1=}vCVg!{)oMmPvR>P*{MEU-~!MhjyGG(5pTx+96 z4D9RMqgl_A_(@n^&d)Rjx)X%p&;KsopRe`fvyMZM_SjRnM50P zH^$4l+S-iwmilL1oTgq_UKP?phDSzvo$Ze(B`r;;|6Jumjr_0&j~Ezacz%wP^IP1N)|em3(TFs)zc@bw+X32n6v|nbnJLdB zHyLD>;KAU0?LqKguNI!>MVM<2%A1 z0N?HR6W&GtXtmFF10<@qK0HeihZmHwXS37FcOl|_04u*tXFDMV(dOy*$jHR`8owv3 zLXMC!tz#v|0v4lJbb&b?pPXP}Vd+-c2a^%pyh$$ZF3A7=A*uh2{r-Bz^kZidA|fu+ zA7AP`P6){Pf=C{J6b9)r`6DyY{lQlbtHW=~EfR;R5l}Kt9Z03k4AAoPGKqD?!spql zETrr_uKO#D)bJqX#N1pqSTF!S?jfJ?rn2erW53-#IMRHRo<3mmxXqt1+=gpWSmSmY zQY}Z_a=di;6uk@T$jz}5W@P->+m{{l5JB7JA78QUUcG!7b?wI7pz}!n8=#<@HM<1J z0t8gz{K%m4c1CF-f0_o2EzsY;NG;oTBkUe8(aa7HUY;VgsPlW>8>ga$`oIy69)K$9 zgLO{GD8_Sk?FN>b)3z&HxY(q&zZ$N>`UxoopCcLzgTfbiDz{Q^5wyCZI;HN0e%>6+ zdbMU0g@e}O#u0lWm!9-OXRI+2*5RdreQPK(-27K#;A6qNt8lmD zz2#rO-onVq9z1{V?(Z&eE2`8p znQ(|eB{yMojTFf8WI48m-Fxxk#gPjUR#mnS>Dw80(JsHPEY+;rH)^u~HQBF8;E6#L z@nJ8)Qs_;Q=+#LUw8h{ogXKUiujV9!!mL-~0!9|cFr9v%q9Q6Ps;^M5smfuswXIDF zp@u=qrun?&-80n|W9s%;0xIFHEpG}ENe#*#%b*n#v3lB|9xB*xWs1(pU zLVSF;kfBsYq-0M=XfntIoLv*fI6syapbEg*6} zc*A<_cEPKR2%eGk7yMsyJ|BnEVzU-DAV4fN{aN*BenF2MQ3gdR%YF(v`kV~xpOjro zwGDwSH|@I0PD4ZE<#hpq2n&X0b86;IWjt)`hVqFDYh!eXSSK78hz*ND;$*D^RHJ=a ze%x;xuADwekjT$@m7)Co`}egOZh!S`je_$?uhVqx3M&gAS)cUdB3pdOQ6Dyzfj87r z&e72^ocxhy*cum6a2IzeDvY)dG~wgmt`wr`{D#WZAB4VpHx0{r|wg6%v_{SbF7u(`Pz*f$BACQ-F|{y;#*6`$9nTk1LL;f>lJ zrQ-aB%oag9?VX*%9>;um)t+ar<%(V(1uR62IzIwkMkb3@OL}8a3EL-#dkE{yTF*54 zqhX0!=_Qsy!>PCYzJk*CQU`@y+}!3)-bd|@78!Kjc1#C(Nl@65l6ty)VRf{!-j)17H$r7cmI&`ALZPIB*+H$))G# zC!4Hl*XqM!&`OsAxx|}%2x4n%3uLBr z?XmUvH?rA(@kBw{TYd+QVG)f8!D60v?_HFl4^&OwS6|m=duO5RwK$%S;4OoAC^QlDU%IeKeycL2HECl7r=mjH5-BCGJ1o^nirnKTRBmlNlh-L>Foq+ zQ1zMdVV8PcoLjhnAck$x;uZbn?yO}5bRWxUf;3Nt0b9v73W|?a_MXJ#5xKq!Y^)($ z({mwK1CxNVea9idIcW}Z_WftM+;2T?N31-@)$~?Rr_0{o-PHzG0FrnyF)^?z_67cC z?TyKX21x|eZEr~r`O9h|?XNv$-pG~A=qH+R+?aF$&YOf)-H^<3KhU`gK9MSeg_it? ziBFzzx&9?euztTcF!Oy$njC>rAs36Qhq4*{AO;Z40M&3hJ=$iI_u7G=nXgJ&fm-Gk zgUm2cHaW3V>QBFCgaQk`MJgDtk{?7UR(Fk<=_%2sA6IUY6vWj|2F~I@`Q8kd=y-{b z+&&naj3ybltDUECx+y!H~+lUVHzOksb!La3ljQEFYH;vQXCv<9Ev+5MI^+r11kdRsN{R#J* zkn0F=Vljj79>j}~G5hu{e0!EeL>oydRQ%)v{i!X(8UBLp2FM27Jv=~$*r3D2#FR!Y zp5IQLS$-uo@~@TV+FGfr4`0IJL_~cR*18G_sgUR_UUqhTJ4mr1PFHV7N0vqbwLlg! zAlkBKj8hJg((c*LJcjgMhMguW#M^ z^7JZg(hHu^0@o2owV`Sj?+Se=KPOP$zr z%d65U0301P%~U`4@oh|Hf#NhiTw8^3d7d8e7NncW0}vMk zC^r|EJe^vty4XJ>0030V7qfQpL1!F$PIob(-~(GW1}V+JpYTw^RV*6!E9==2Bm-m5 zx%Gs)D7oHBg0!YIxuuW}Y+R7UCZ8vi7DkN=Tg59iA5pgb{#{wn5|@b5jzdotZ*KBG zLHD`%%|=f_JzOGy1p3|L4ui6)X8uB<1l$fR7H*#W@?)2L0N{ql&0I(au0aeWp-?;* zJqTKvH(-F>(UTp7uV!Ck@9}FjAjkOU^Dpl>BxZOU-*o^?1!aoF)Bbr94F7@-jN}JX zx=V8TDEK11qqCD-zzQNMDcPSU2Z$AN8kzz&x=Z>9eIaQ8FgpfmBqL>yaf*jJs3$Ku zrdqedAq`!R8Go{mjE1=&R)UBH@-v%_@4i?7+=xj`>`({g`S16h0smQtUZf~@KR
    tBiVQX!xLH@VTJV zVCD-@eE~uf$nG*^)HoN}U29ak6q>(Kgk12-yu3UJ*TrT-*>!bw zn{_7~gm*z1`K#=I!@GdP&wEo{9vP_u6tFd%qAgs=>--eFMOSaX{KKF%Oz*PX8!fk_ z(5~}56aL}?P^ykS>efh_>2gbL)iSG}!g(GoEiE&QQ=`1Sq9uSdLO`e8HpSF2Yj!$W zM=5brQ&Ry%^a~x&se7=uH(z!O)Bn;$HYNdFKqKPF_UO@mxWH8Ai}cnB>w0{W*49=K zIxI#Dy-7BvYDL_gR)_O|rvRuG7A7VS)y2_VB&s(BxLP6TBHl`(m>WM&k>v+^Ipt*0 z;Ds+=zFfm1C{(nEg<7Vno_T=f0cvjQaC2I|1F&Ji$R3brJ8!)O930@Xws?eu5FiEx z64JaIQcmQ#!8pdF2f38`3=%E`W$FqJ3-fEgO-wx21^7#kMn6E(dN%9E5ndx@RS#oP zLjpq`Dl(8~gH+k$AVI0c?#Igcd2aF^@4NQh7LPZTHdoPrzhm=ii=dj_hhR=QzI(^S z%5T#5IVy^%8d_+TR%@w~#?ngFkYd@(@a+S#XnZ3IPo)(5G!1 z7$6olK%E0h9v~sXH4PvF0h9l=u;6fU?opGgmO~2ZRtJDp2fSgrFKsxX3*25Em?zvH|&CS0iB|vQLO5(Sah6nK#O%~`%3 zCItTLI|ND}%ZhCP+cI)!7jjRUgXo_;Q|l$ zaN`%GasRiGgC}kQSE>qB^wy=Dy;Ph3m%o@<12}g%aPi{e;viNj*}^2uza6w)`xRW8yDpIQ27Lzn;v4+OXBw?}pN z<>~rY>dkX$9Gqv~m@oKlT}pc%!u-WH!!ylwQYcmG&NDz3fjBdQV+t72B`POn6Cycc z5P#F9kq*-Bh!F5I1j{a;SU&(-;k*xfl{-Z7b+zB<4kuk~&^oKL=22}wJ{xyEwtax@>SagCuJ>Ei&!7U&RB4$6k!n&CGYUmTu{lxxan z`nWc=f;O|%CClM~03qdH1KYj@U)>+iRqP#>xL6>*WmWHW{b4e@A?(NkkJZL>xVhEA z>|{TIMXOsg7`9pm`Wc*7Y3RjMO(w6SLJ~2btydpV^O-~)!7^uvVAU=42fG*4I}!BJ zJwphbZ|2p#L%4j$PcZHBPkwx9d(vAM+R}TSky%0P+Xa_lJCRYvJ$!#?tt&E1y`o6E zauTr+dU7sUnj-oW#w(|j>(J!tSz8yaQF;1nzkO8poM&lu`W3ZNPjIE`+od_RH8I0? zD9BT1L4y(;-s(ZET1+@pCiYLMX z8WAm8wcD;0BW19Yr>0)Ev7#R5t~TihweuvM@RI^LFVW8Khp@%{%Pk35K<VE;HLU9jur+rECj z?E+`JoZc>fmw5zgm1tI`;w7&9F>7TXV)!sb&WpSJfVa1u)er+-xJ~(G%wU^l_Tl+$ zDGoN4MYl{LGUj_7rilgY$2pDBG;5OH%F~nwTtWi8BE}7eZF-}|5)n0y0_$h{#+pSc z<;~KS@Oq(_mYoIIRhwVmJ(!Bd5*mTip@hBc#87eUy4+S9^6DaF6Y9CorF7`7e@ada zr`p$XecJCjndcv1X^CF&_WQUu6lmSK>v9{usuvPk0juhXlRv-fF{eYmvdtyrw28j3 zjIb=aiK`MhdCn!ax_}5g%FdE#3rJCTuBwD|&}~N`Qy_haAv5_T>Mb>ge87d)DKy<_ zQJtH7C9s(itRiGP=b2R`LXxwESzHV)hT0Wg@h--ACtT4%c2Z=CAav3bT5;Xk%R{{u zhlyN_u-1&?UI@MK^pFq4@~l&_UAv-JkfsGx7C3V zE69uidDg)@GZ`bWu^Ufg8G%ULI~vC*P5$)Zk?DX?5w#0Fs){qhDSx)a>ioH49m4Eh z=#QLf>e~2EFAihGTH4kg@L8;at_MJs;8wHD(= zIEIgww!@1-b3zdXvRDoDTcn+~)1;d#hg%4GO!qy?*t+^^PyrnB&Ip1O|LY(pgNhD= zT@hCu^^sNx8Z-)=e74DNtJYQ=@$~HIt5Acpra2Sj*`5W2p!)7$w5YI%-!H15QB5GC zt5S0yCbUG+=N+A4B17iuH*7e>oiE91%ObPkh|skuawY8WTjnwYI4sK##`LDp&jk9$xYJ;ly-{_h;j)z-z{STI(9`^l{`$-T?vF?CMmDaB^`?(zW8 z3y_?^mu9iwovvWglIL$lGr4@%T0p}rqiJJjH-w|1ae4e`2W+mT(1 zJt?>|{N_``0Og#jjF0>HNr-8!07c~tf`)oDC$gvOxndsA5rNHD`k^yJ!r@MX7=O2d zjiZT#MZuBo->Y1mbb2-rSKn8!oL6NoUqj~_$DC`ao}oT)AV$dsIbj6)4+aYFZn_*P zq|#TYVlf7}@FH6X`3U;?>$PZBk&%zoykQRxx`YWsYRp z_ul&xIu9A^VXJ!Um&4pd6`SLp=95*Ib1}~r+#>kTVm2YKTu;5clqxoKOLPWS9+?4z z=^&B3S6riBq1LfBR6M!7f3##FX6Y3xH0tQ>}k;2#}+5ONTmwDB7N$2A|p*RF25S&0O#`r(=nZV-Gg55flyj?8@me|tj|XYuR2UzTPy(ey8t z*~5XSpZH9R4LfcdS$V$Qr>lJUXNsat9OI@HagccZGw1Iws>I2;_5I@GqufxsMGD>~_YoYFdpJ&H%ip4`8sDcx6XMz;!K29+b_S=!J|F} zy$uKdYHUDQPVnn=)0Yx}<|2Fw{!){EjLp93X$e8WqaRL`F-UIm5v{svfe#7U#vV8%d={tvg7}u_W5}sfnoJJzqxDf%+;9s1pLpgzh zmo<&e2H*~zJ#~N>5l{`#cM4h@K>IC?gl7%tba2)M=9+bG zha5F=$;rryVsJ`*D^%C#K!qaguyV(Z#mNe^j8s%OIXeq-iUMLD6k~}%mB4Aucf_IINg9^rsu2Wlgx}gEtx1)rQ=T~dN4r9GYlv?;%?bOyt<8|t- ze~H}X2p$!APtVQH7ES_q1t2J_N~2ecr2gpRLFoegR>Tx#_XMb6Wm42Ormbq9oSZbD6b>XXr!z=oT@CJ7}@qv~ZBM*a5= z)w7YYPqX3N29H%$@7Q~7&&n;BiO~LbEJP>NU%q~w8Hmherlo!R?Ck88;cKTkeNYpIQwX%)qsOQKE?Ea*Cs$LQHu_=D z@cvh*?+sKkK;ro^gtw!1;SlXscYyDx)w#R06nx+njK_}=^g0*qlCYh(UuA^Mn*YQ>y5h8qvE3NRJ%fwI$i+W);O+fJ$>2@oWrQYV>wvP=!g4) z8mpMNotGpCIQ7g^xYN`J8oRpmae4LkDKVf!Ekqz_X9H4c2LOsC`Gt`Be&Op=B1cy_ zUZgbxxQRVcoDe`ZteSoPt?AZgKJCg;1p0+LIZ`5Cz+DAZ#J-SN@UDPV}B zKS4=EUJ7bF!fu>kLXBTHA@jbk97gnno}lHmM785ucVJXPLM4DoeQw|;i#RDGkXC=9 zW^qOjepq)u0o&4ZbjM=07nL;b}wITYe9a ztX@**qs7p1w*&CFFo5+yEPU7{gzuc=>FEi)yLx^hvwli62<1haSLxe;ngIx%mx<}I zEg`-X2S(y$(}b~cFI;3~U?8S|=K_f-v)?M`?CcCMvgTFLZ*gmeTF7o5v{(TaV(0Ks z7huQtC*Rz1GK_$Z0Y4MVKu@o4U@#_D3LNfkG+OC3gRhFx4Ptj!D(ROeQ=otZ6EnWb z6afbhtB+@UeLFgu5T>Z4)MN+94FDYz)o+>O;_+@;*fdj+Y0Z(o9a4r1$FjV_jMd0d zUyo4{*K4Y|RXu4pMK5QhXhux{N5yYBK3Jej zL-&QZ4T1I<%>da7h{PW}&yL5IWhcH1aCBd%a|dl5qm_)t4KQ)L+q9D)h&7b}L{tt1 zUaM+fXCJSgbbBlB?jczc?i;pvmURGC}u zA6YdB%wGib@@AEBZZA3CQ^@!f+Yf0`yann(#@QVEl6+cI^*` zthaaQum~QbdRB9bi4Rs80zn!EI-*KVn{*x5$797{J?b|)l2??Klza`*VQbIzKzo+6 znilmSO;BF_Az+$!)j%rH7GUw(TsPXr-tMm>nh2?>?&!|4b`T#^Pv2d1P#?oN+5gZ4 zQiy)U+6%H)&;>QxG-eT7V`y*Br#!kK@K#1zTE}&nu{?J{YWtoN`dZ+zTRdUmScNss zih{CoTW73A zwanpVpA6N%N|ju#r&O%6)-hI#vN4d)SZ+ppmGO{d9(*VQTRD@jaDa6IPIb{}gcJmp z9-S4@)O;^XDdSDhE{n<$KieAs+FN3j=b_Np-*M}&nl11X=MUd0J!+Q8nwgmqob%No zabhmDl3JR?c_FA_ULRs2y{f>ka9B^dQsV+wlWGj7%KP>TexdOwW{6F`EHcc0m~Kg; zDsK+*Uf1gM*a}dxUUtt3wZHEARUu`jj@{%H+|@ZDmG7-u+9KP^g->2mQgs>r^>v1Q)AL?o$NOD#=b&hAhj{R@(J&zYqCqAL}dH1e7;aY`*}tu;b< zAPaes^NEH2;KR(Hz3;M&9Gaj0mt)0?;}y@5Xbu zNASS;fAj(+Oq`FrQ@XRg1tRdrYK47LzjHW|KUD3I74+6i0cro4yU{U`|RC| z>~|qS9)^OjU7~KUc5OaS(53XJHn8r0S}98U)+$;zLbh)5S#|~O-lpFJSWcC0hXrcU zKXSczY_=b3nmt_rlDm^VUhbjbJaG=3$foxaM$Ae_ET5n8S%C$>H#KXA2P} zJ@#Kwa1S7aogU6Fo&I~6j+xzy#ZN3*`yXa(l4S}v=&S)^&)ZxV&z6Xzu_1mv38Xa2 zPo4lO6_@1UeW#dVc5Ju^-8*3o+YD)QOr5g}yUa^l2-cPR=R9BOP{>@KCoZLr<==hv zazU9Oh?CLWY7ia-{o7t^_a=)iSpBHFkqTvQeu{dyp}G{N-G}B36b5=3y97~4Usgq3 zsP(80GMOHJ@V5DHX;`$m?wy^4&1?IVW{jkb(&toOt1j0U$^0sZpVxf&KgJ~1hc+Ze z%qEBzvqlJAM>oyB`}6`8lckmouX`hzliEqz_CCC^J*AufiLfg*1C%l; z!==-!l%z4G@);f&1+*|5$ff}s#yOUuLUW)pRLcLMwBX8;MdJn^9+mJNa}t%W=0mOV zB)xEO)vi`G>m|5jnVVqfYA3(i(Bl5vzW+TS;Q01;cE$lq$$IeX`!#FpP0*#gY(a0D z@WP9jKMgZ=1LQ=X-$_VFoU2x;TkaUSz(NIZY}0Qa@Njb%fgX^xREk@FGvnW$)DSdX zrL@L9HCqK8&zHF;qdn8bMIW9mu^MmZ3kwVszu90=!P#sm=TUPdD0hT3c*OdBDa%Dc zTBiCn;hpzi*D>|L#3SoYg3EWLd-r4kKDQN(EvV*JO)ZE>Nf+nluF2`6V>NW{3EI5P zPclh-VH`s{5awiY58M%TJrz?(SGN>Kin;DR@S`Rp3pC1f- z10M4bW66CY`(&`<-UqW!y+F4HxEKofR?tx3l%JIO3CO`8dfCHWBdGD8*43?Q_cWaw zf6p#Y=5#AjMrRSt|F7}h(U!NMZZa=lwi#)>-4}TADE*x2$oV=ZX0cI+y`{HGSdLbb z{%iLQ_wPP8K$0l?4iL}_=$n6?ywbNmNr%~W&}jY{+v$$!V7T(D1Kjz)8JYCpxkkoy}0zKCs?8-9Y5X1v29pnUS| zSxfv(Ud5C?Iz&(-BY;*fd6RY>-R`g<*d=i++vWvJ3+hg)ZdUJUwMW z*kodEJM#)q{(hcxB-0x6>-{YIyz%+2Y~Fc-yIfb)f}Kw1|2oc*MCYWd6`;ElzuaFf zk?$)rxN<5Jg;N)@VA3PcA&+7zLy9g)kqdltxSWR(T-Is?Xp1|MB1*U){ zEqV+HU7zOOB`ySQVELGmG7YeTSk~)*4|G+f2LJGp>5uvS5#1DCV*+s+A#8eIOCsvvGBH-eNXk zdiH6r*X#V%4^QVeJnObcF<=bbzxMWfdd@<^Z106o`Z0>uax_?R;yn8%;kq>BLM0*6sPZG`fWLv;}l*@!G z9wc17`yCcr0Gul6=K=LLoZph%XRNZa5;QwVe*<@^EB5+#-KBop$$QmIH=bE4+0lYK97fm;VIFG^^#p4 z%I)iVyqD8UwtuV-3kuq5U+=K!uve%6?G+E^R#%f^WB0e)XzF*7BB%QKU;l~$+?lemAd_OS zdW8BjMtYU<_Rxi#r&L z60XUdmsSjJ@4gEP(Gv{5O7$Dc0at?qGVfvv#RDS$&reZMq01Y+mtPB9#>YQHhb3Kl zd~oGANh&g->xD;h61e^9*YjDvdm&T2_ZfKE$A+`<=EqC+c diff --git a/docs/fig/intents_objectives_kpis.puml b/docs/fig/intents_objectives_kpis.puml index 67779a9..444571b 100644 --- a/docs/fig/intents_objectives_kpis.puml +++ b/docs/fig/intents_objectives_kpis.puml @@ -14,11 +14,13 @@ hide class circle class Intent { targetKey targetKind - Priority + priority + active } class Objective { name value + tolerance } enum KPIType { latency @@ -30,6 +32,7 @@ class KPIProfile { query: string endpoint: address external: bool + minimize: bool } Intent "1" *-right- "1..n" Objective: objectives Objective "0..*" -- "1" KPIProfile: measuredBy diff --git a/docs/getting_started.md b/docs/getting_started.md index 5ee2d9d..7564636 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -117,6 +117,7 @@ sections for each of the [framework's major components](framework.md) as well as | Property | Description | |----------------|-----------------------------------------------------------------------------| | mongo_endpoint | URI for the Mongo database - representing the knowledge base of the system. | +| log_file | (Optional) Path to a log file to config klog. | ### Controller @@ -185,21 +186,22 @@ Each actuator will have its own configuration. ### cpu scale actuator -| Property | Description | -|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| interpreter | Path to a python interpreter. | -| analytics_script | Path to the analytics python script used to determine the scaling model. | -| cpu_max | Maximum CPU resource units (in millis) that the actuator will allow. | -| cpu_rounding | Multiple of 10 defining how to round up CPU resource units. | -| cpu_safeguard_factor | Define the factor the actuator will use to stay below the targeted objective. | -| look_back | Time in minutes defining how old the ML model can be. | -| max_proactive_cpu | Maximum CPU resource units (in millis) that the actuator will allow when proactively scaling. If set to 0, proactive planning is disabled. A fraction of this value is used for proactive scale ups/downs. | -| proactive_latency_percentage | Float defining the potential percentage change in latency by scaling the resources. | -| endpoint | Name of the endpoint to use for registering this plugin. | -| port | Port this actuator should listen on. | -| mongo_endpoint | URI for the Mongo database - representing the knowledge base of the system. | -| plugin_manager_endpoint | String defining the plugin manager's endpoint to which actuators can register themselves. | -| plugin_manager_port | Port number of the plugin manager's endpoint to which actuators can register themselves. | +| Property | Description | +|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| interpreter | Path to a python interpreter. | +| analytics_script | Path to the analytics python script used to determine the scaling model. | +| cpu_max | Maximum CPU resource units (in millis) that the actuator will allow. | +| cpu_rounding | Multiple of 10 defining how to round up CPU resource units. | +| cpu_safeguard_factor | Define the factor the actuator will use to stay below the targeted objective. | +| boostFactor | Defines the multiplication factor for calculating resource limits from requests. If set to 1.0 PODs will be in a Guaranteed QoS, smaller or larger values lead to a BestEffort or Burstable QoS accordingly. | +| look_back | Time in minutes defining how old the ML model can be. | +| max_proactive_cpu | Maximum CPU resource units (in millis) that the actuator will allow when proactively scaling. If set to 0, proactive planning is disabled. A fraction of this value is used for proactive scale ups/downs. | +| proactive_latency_percentage | Float defining the potential percentage change in latency by scaling the resources. | +| endpoint | Name of the endpoint to use for registering this plugin. | +| port | Port this actuator should listen on. | +| mongo_endpoint | URI for the Mongo database - representing the knowledge base of the system. | +| plugin_manager_endpoint | String defining the plugin manager's endpoint to which actuators can register themselves. | +| plugin_manager_port | Port number of the plugin manager's endpoint to which actuators can register themselves. | ### RDT actuator diff --git a/docs/planner_logs.md b/docs/planner_logs.md new file mode 100644 index 0000000..bc07501 --- /dev/null +++ b/docs/planner_logs.md @@ -0,0 +1,200 @@ +# Observability for Planner Logs + +This document outlines how to set up the IDO planner config, logging utilities and observability tools in order to +collect, export and query logs. The logging utilities (Fluent-bit and Logrotate) and the observability tools +(OpenTelemetry Collector, Loki, and Grafana) used here are provided as examples, and the same principles should apply +to any other similar tools. **Please note that we do not maintain or support these specific tools**. + +## 1. IDO Planner + +This section explains how to set up the [IDO framework](framework.md) to save logs to a file (in addition to the +standard output), along with setting up logging utilities that handle log forwarding and log rotation, respectively. + +### 1.1. Log file configuration +To enable the IDO planner saving the logs to a file, add the `log_file` attribute to the configuration file as shown +below: + +```json + ... + "generic": { + "log_file": "" + } + ... +``` + +Key points to consider when configuring the log file: +* **Empty String**: If the value of the `log_file` attribute is an empty string, the planner will treat it as if the log + file is not set at all. +* **Incorrect Path**: If the specified log file path is incorrect, the planner will throw a panic error. +* **Path Types**: Both relative and absolute paths are accepted. +* **File Creation**: If the log file path is correct but the log file does not exist, it will be created automatically. +* **File Append**: If the log file already exists, new log entries will be appended to it rather than overwriting the + existing file. + +### 1.2. Log forwarder +The log forwarder is responsible for collecting log data from the `log_file` produced by the IDO planner and forwarding +it to a centralized logging system. Below is a sample configuration for [Fluent-bit](https://docs.fluentbit.io/manual) forwarding logs to +OpenTelemetry endpoint: + +```ini +[SERVICE] + Flush 1 + Log_Level info + Parsers_File parsers.conf` +[INPUT] + Name tail + Path + Parser docker + Tag kube.* + Refresh_Interval 5 + Skip_Long_Lines On + DB /var/log/flb_kube.db + DB.Sync Normal +[OUTPUT] + Name opentelemetry + Match * + Host + Port 4318 + Logs_uri /v1/logs +parsers.conf: | +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L +``` + +### 1.3. Log rotation +Log rotation is responsible for managing the size and lifecycle of the IDO planner's log file, ensuring it does not +consume excessive disk space. Below is a sample configuration for [Logrotate](https://github.com/blacklabelops/logrotate), derived from the +default [kubelet log rotate configuration](https://kubernetes.io/docs/concepts/cluster-administration/logging/#log-rotation): + +```ini + { + su root root + size 10M + rotate 5 + compress + copytruncate + missingok + notifempty + } +``` + +### 1.4. Deployment considerations +To ensure all the components work correctly together: +* The `` should be consistent across the IDO planner configuration and the utilities for log forwarding + and log rotation. +* In a Kubernetes deployment: + * Consider creating a shared volume to store the log file, allowing it to be mounted and accessed by each of the + IDO planner and the log utilities containers. + * The utilies for log forwarding and log rotation can be deployed as sidecars within the same IDO planner Pod. + +## 2. Observability Collector +The observability collector is responsible for receiving the logs from the log forwarder and preparing them for +ingestion into a log aggregator. Below is a sample configuration for the [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) ingesting logs +to the log aggregator [Loki](https://grafana.com/docs/loki/latest/send-data/otel/): + +```yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +processors: + batch: +exporters: + otlphttp: + endpoint: :3100/otlp + tls: + insecure: true +service: + pipelines: + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp] +``` + +## 3. Log Aggregator +The log aggregator is responsible for collecting, storing, and querying log entries. Below is a sample configuration of +the log aggregator [Loki](https://grafana.com/docs/loki/latest/): + +```yaml +auth_enabled: false +limits_config: + allow_structured_metadata: true + volume_enabled: true + reject_old_samples: false +server: + http_listen_port: 3100 +common: + instance_addr: 0.0.0.0 + ring: + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /tmp/loki +schema_config: + configs: + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +storage_config: + tsdb_shipper: + active_index_directory: /tmp/loki/index + cache_location: /tmp/loki/index_cache + filesystem: + directory: /tmp/loki/chunks +pattern_ingester: + enabled: true +``` + +## 4. Log Queries +Log queries for the IDO planner can be efficiently handled using a log aggregator. For instance, with Loki, it is +possible to perform log queries through HTTP requests or a dashboard. + +### 4.1. HTTP Requests +Loki expose [HTTP endpoints](https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-endpoints) to query logs related data. For more information on the query logic and syntax, refer +to the [Loki log queries documentation](https://grafana.com/docs/loki/latest/query/log_queries/). Find below some query examples: + +* Get all labels in log entries: +```sh +curl -G "http://:3100/loki/api/v1/labels" +``` +* Get logs filtered using a label, within a time interval: +```sh +curl -G ":3100/loki/api/v1/query" --data-urlencode 'query={

    8-p`*59is&nPlxy#!M~^xO zF5hPvtA9+psivl8QGUW;y@~_JNq9(sWv|v7+p&p>iLA19e_Y0;*CVTg?$#$R_5PTr zP+ryiWeMcZMG~k$hlYjGsbQjG?tJ~I#?8Ycw^UP8qd?Oz_q02uz$`7bL6#r_nZ_ zn9j~l(lG~A%D#1%m1!lzAavy8zIJ7U) zb#u83Yjnr=$+zA2Fge<}z6IYx9oYGOWA0!=T?l45LRE_4V~ZP;+(LotLV>F?wg;(;dhvRSX30x;nJX39CNE?)TFh zNH~}*3D$)35*->IzS<-@n3$Zr3Y!H^?(rKRUtaNAPwjz|41tJ6Rd-T?zwX`svymre z;(Xb&pYLFi;UUUIopLnHaTCLXsfFV%N_UW%~#)(tH!c+BTu z4#%OpqoXFTv&&a{Zn!K>2-}Xo9qC}N1b@}Rmi}?elv$}2Jb0v$-Z*Gq- z>4)z5@h|w4nhn2zn^YtEUAS3^wX2KMj+7P!^^SWxTF)fN=Q*2 z1yLt&U0s@J)7HC1e0VVO_>Gl`gy7~#QR}ZCGwN0E+L-MROwKUV?U8}gVor8?qm%lx z)skn;1Ln`2`x6rrTL-4kZ{G?MiALTWWOFCrkv6YP>h|c&^By>`GTR^7v#~nY)wPUD zur#O6<>=9^@}UYB)KBAR%$P{pX}rfQCjY>;VTF5R2Nk!H2u(^`fo(+Q=_JFl{@$qL*`zsh&I6Utk?mW=qX96S{JC zBj%WS2+Bw_E_^)tw@=gcI}EyoKY!krt}W#DcJ|=>IBmXFe;UUC+l05b(H=8AYU_0x z)E;gnj@XIy)w!6EkP*Dl5+l`2Icjq~H8pj8xc{!fC9?x}gGO2iQ~YW#G1*qo_dke4 zmYzm+dqo8Qo->Qag3ND=Y#58d=d>_7gTUCoEF$Hee*Q&p0QeWQ8dYynUD*8>E zp2^*qe{Ebd6|lq^o6!1tz;*E){Jkh6;Z449kRol=+Ny$M&{mLJ&}ni33~f5DohfST zTvDVH2kivJ<8fU6krceYF2V&_JZ`nz{4i>+boIw$C+@EY9JCUA{`=tF;27sJWWi=8 z#kDfCc?ZcUbQ;Rux;^~+yePihprxgiL6;ZCC17%9sin3CxAW&T-+xzxANJ*V;jW>f zu{zV6EcA&;ej`z;A>mTyhD?ihs0TXXtu5XDEf{o=>-DTVesh(>9ec~dZu-+Lh?DE} zQixp99^RRV6;*}#>-`+g|2fdQJIIW->n+j!v8V67d-raj)J5oWwh11$xz0aeoqKrZ z?uv{{f?IoZYDdc*_lY+)duUL{E(Li)CV*L%OU&G;%eU-hcgM~767aMY0GeoIXQ< z9v)mr^;T7i9p-RwZr=y8b^$X zUu%FAFeWMm(Pf>J7WH>iUi=kz3iIY~7EZ75|H?mlvZ4SL!dYHzD|aRw1Z8Q)pW_Bh z#&>^sykHQeqN37Ubm!2fie)J!e2C{cMb~EBcqQRXabi8_-l1jYa^NxQ_i)uNf}hR< z5*!H-lHQokNqGD2c8gSj?U=ILNbLTq-ho><*Rw}v?xL^zs4#s0rdXdCfT)s-jUuYP zfw{TI#5V!w@@#NoEATrGVM~@j>Q3>7_H-t6zP*ym(4B7?i3;{oX^KI*zx5p`1 zUEp@)5(=KWE&KU!NVWI3^+_4_Rd--QnZjP(?75M(59^T?>lLGl+_Y@>CL@86xJI?udq*zdWcNKs(1pvzvqQ2RUMsYW#LC&0Zs-a zzIT|!Rg435!m=kUIy!TeAs&@BRXVgoKmqI38xuZc$*~_>v!jFkB<9@a^W zF`*GKD0%ulrcjK151ugsUK~e;FJb19A26YhDl%V{=5>4j=kTADx~#VANOE(_AnQ1v z3LEca3kXUnzdyoIuN0er5rHFgnG|VM`5JBfb-X+KB}#nPH2L2Sr>><1!do+hjipdZ z&o~+1j~TjM-uU@2gFU6MTF-80tB!%~R0oHVw%IF&3(G_FWsd9AZm*e)Cc!>j$qwrc z&SB(%rz;inB6=f~Oh-6*c&6KVuprXPhEpUWhjj_j77w^Dx ztuYJUGLwt`mf$ue**+wza)^_&sUS-v?kfSXAX(~~IoMMk6H0Sn*cTrQyDRQgRZG|1 z%|VqsJR?6{M0dvFGipV4lMoRGAFkF3a3TCS5)Z+!X`NGIR3ZjXnqsYnGI@2GJqmwpo`W{WoDnL>a|+2W#RjkL zxo=uMz25M7=g}&jwnU{+i_Vls&z==o4@z&|grM&_`bfbHZ%4OTVYO#kDOPF&Nrv;{ za7uq*Q$rp{ilJtyq-G*NNT^ms(4w>uwi8x8&oyS8%O~J5` z^~9EE51e#VpfJ&fX(Pr35wD`hE%8urlq|KY=oi_kP35nx4J_V%%8C|xjDPy{Tyg7b zV+9=~M3uL@aAWhD;X{V|kySjM0Gw1+ZWk>PFp26(L~cQ^b$Jv3jV|g*>8pFJN|&{2 zc<-*5-`IntL}t(U(4X0jR#3oflaZrFiiwFqVOGAm5^3!&cZ*21{KB#MGOwJ35r=AW#OeL-kr)QP%I~-)im6t}W z@+V~j$Wu>JHpeSzK7&*AxIT=pBm0`lnzo%Ab=25xgDDDjBua8w`{kwA0Qz$?CN3JL z3E_;do|qXWR7K1Q3H$f&zo5Gk8TE45SN{PXyVx?hV~o8P3S+axm60(%rt<;AJJKjg zm}dF8yhQYD59=4@0^axY^E*bb@y}oH=#lst?8Y6vI zbvx_<4KJlvtSGOC88o%`cXRV=rpd1Q10qV35H8_WVCt6UDCE@0B@m%Byw_$Ha`{_^fuurg?bMDD<(GwgeZhm^_ zM{C}%bcpE9eKm)q87-7~zRVp-VpH+4DDPI}6EyiTgVnOws$X2}5+NpG*766- z+$5wFt|__so_%)AFSsl4QKy=Z-vg1Wl zIeybxXOKydkMvIalq)=hnVC87Mi)brsjpUxird@euU&?(uM$5#cyD>8S9>MEXW!PW zU-nLC(8;=Bj2?QMoBe3hl()hC6Q5~gl!W%INu?K|EC|=UWAdafm+5?Ly^Y6k<{3yA zJ$|&j(e5~>XS6zVC#(maODn)xUc|Q`S`}FcB}suH*gz-J_WO;uezWN(h^nypwP zBZQoh@!n`Ig}H%J!hGi5>{fRRuYW9 ztUi~#OT1}UMs)bPC)DqjZ8#`HC#ubeD7Ru$GU=m-4>?gizPO}(Y!iw5`H!nr(g4b# zplsNd>%Ts9kJ?0&E#lhI!#=brNB0^PreA1&MewXpwKEZwx4YGw)$vm6LIvKtNWxz|Koh!WtA$J#rXL6&X^1F#&yDg51&-I z64J&K>kT;0ni)Ee88oM|&}nlIAp+IDl`VTz|M(E^pb;E1H?Nk+)~$W`I7W6p>88p^ zFv+o-pKLz8y0T#NefKSti*^h{;LoBu^3MHiceUkn=YWW!ZMZ29~k%O zSag27!^IH#hW!|3)WtZxeXUX}HI*^DoA?{`vj?Z6_g=B39u5vd69IiG>G>wl?>*$I z-dyv=?&~qkT5IPA**|!Z(OPt6N?G+4LR8I!Xt*oL&u?uqIYyApr7pAcM278XxT(a& znhtUq!Gn`NiZ}EIg?$JhrlFuzC_~nIe^u^F9|GTjDI?o-ciC7oIPTJvyA%Uy@OYPA zN0uS7&DhCoS;eAuNFmr=T*%Vn+u;)(biMs9$?DA*gIfD8-tt`Rt6(PKGuV_~*c?eX z!~?Sw%PRA_qt8wS&LOJwBcjg8(wKhgwY`0N@*a9wuJZ-E+ruuXLLr63?_Q^`91AzZ zC8%M9i+M#jPURWi`}WF}3m>0oV!l=a1=>D3(#6K~7Hgkb(@a1KZWd*-cj^a^63oXYiv67mYf>QZI$IaVKfC#b( zwi$b@-M*omOV|U0qH%a@Yvf>hxg(?U2VQ~c&f~kqq+`a zHAu33wT{@GoDntIv6eLEy9v25shTEZ*CDN!iih@Rq~%j3%c=9WK1?@d;9joeJ2;LZ-3dFoSq#J3%5d0ZF`dO+r!v5dDrN1ip~%a zu%~r1#BYCTgtOh&e{QC6f%!$Me(?&R2{XupB+Fe494^zsMS<$>&d$z2ILPs47;C+} zwC|(mh<4!W+@NZm@9Y~}nH;vX-ZIQcfa;-gSu4!qk*zI&99-egI0n%l;PP{=qNTER zm_QVixdA9=66yJOv`e!(KsTmqWsGV;jvP~jIr7Ph&>GIg6CmexjZ860YRi04iwh4E za$;}qm2`ZC4IP|PS*~_JOPTC8(HfWA+$wHvQbw812r zD!JNbyQU8jrNcFQ^0!Z~B+Kqtdt~W=x36KfHQ_z)cZ5rsS2cR?F|!M z(5;|eq0Qt+e3B(AGxHk=3RV0WHR5-say#bm$M5%}L+N|yD*=H66{8x%;(uz1EM(Jk z)nw-2f(q`Q4{R$+E3V!3QQp#}S>?EOs7Sx+Zr7e){3!pWIm)H~PM06YRH38BCXy@A z>~r;zQkB#ih@>XH&k@{c-x(NqroceJlwy$TBpjQYgdK9UQU;$u}0 z=`16So-T)8k9P9WVmfZo8T%6dHLJXS9?Z`PT-c0I;TURAfv9CMNZ-%^UfIO-(h=7Zf6Yn7vpEdoiJ_#O%z z^D2NB8j;F?4)+wVmzPFL)~BAXixD@^LxVJw{#hZIT|Y^gUI+6cIazmBfnE(0&g;B1 zp%!<>yVt7N63|2Z;LXd~j`LT=n}A+ue-{0b-WK_UBwJ4R$w;6{Mr;O&Ma#hNe5pAy8TGiCM5gK1YH`2?_)s!)NlfDp1=K}4OhA?gwanLNEk zdP6{}o-ZQTtnExtvM?~H4Rv*G>rE>O@o$@+9q%o3b1^U&Xp9hXS)SrqZ+gOG=NaY> zNBLnyL_|mkm-wdG!^36FFO)(L060uwOjy3*Dtb}k+C9OK^Pg+XkJbcp;LC3bg!{%6 z78U|^s1cN;cP?h$mjiW)V;E?&;*cu$@9hzW1lE4 z_B=zkWqO_+YZSbu!Nxi-8wTe#0o_%v+P8Of0Ed@)`f`d&xZue3p^cRpl~kGjkJD*k&WD=54p~y&gSw0 zaG>&ZsGR!)1xL57-{#aE9li*4WAC%$Hw+wyy@S{!o(SqYkG%6+Ck>^#mVER69QJI) zQSZzz`1h~%*imPQ;AL;ch#O`fgX9<%7B<TC8y^ylOQc8G7Cr(aI@}7SlJH8A&he`4Y`C|v=YSA*2DHl`KiM?;k z!n{J^t?`b1eHDs3B}K*7{NpZR>BHl&hizSN-h4@k3^UnUQedy>EqmTzqfBzHDV@IJ zrJH~AGL+{&RICS_v9PV51Tgpqfim9aa_+=2U5U26u)L{KlwMoh&Re%B*E)4Qz2{hi ze9U0X4WeA3ruD3ng?fT?eVFVYUqDV;N#y8J+(LM{SyWGWhEATk`*a3OP<%f~Bp|c; zb2xq-DBX}+?Yj-cpY0u~syrsigbnGfmO4f6o{3Fz7k>f&}=BtOg9ZD(`M+0?TP zq}opCK2ap3q70($5tl-X4{3EC3I|^?<1)4nQdJw8(O70Ehw{0H*X^lPWvHZn+KRk` zpIKNtsGJ=Z5_0r<^K%IBMUg4x8D{p{$G^q^{|3nf3ZZkWeT=ZOP>+1v=@C=8^3jw~ zxDgCnx8Jyo0Vk zltl4{iR$9_;+mcCuK$;;gH$&3modiAiOv?@B(~X}U zM{6I^1FnP}2^{&TgJ~IV{p41O=Zmp+N$L#lZ3sAg|kw9cQVK zaM+x_L|l(Xtxx8@I-MK?UXSZT3SJx+c1U5LZd$saCH0&UWE{R1DsL9|K@iR?TPL8m zi@+8EJe}g$Nhndz`z;e|Q=10YfpOJRrl8S-r=x9xY`dxk7WpKItS{f^sv z^!A|i7LKB1G24hMsdrEaN_lt%%N)Ftud+k$g{RSZPZf1v6@&M|q{!G)n7c-AlSm$- zZ;|pe+WUDlT%>7KvQe6{scXRO9wwn9MbZcqE+O_v?~3#47Tj$ULGCF=CaAjKxrFt1~IBZ;7|3N`2OXBy?FGkdm$r*b|G9SZ5Zi*!T?(k*?nV8o4vpg*(2n7?^R z9Lc|#7`}OER8MbF|KD_-AwE!2z+K?Uj`Cb>a2>M!^ZUt=n0q8cXPSLuSdLpv=#!U` z^RT$zTI2U97=FX5{M6E6Bxbwo2N6uiXnH=KHfC7$Zrb|JIM3egAZHFS;YJoT5!O|= zMZ=Et&b~tuW74jplk2oCVGu+42oNw&`ji*C(pU7N`>1#t^Cjk0x0OvXW6c*dbwPhB zJnRAE3>p>^G$cl!kaUnC%b7Wk)$qu2-s%3#^#cRQh*lKvmLA5KfClYtC5Ii$KT?+c z@lBv~{Co_^%%A@n@HcrFsOuUuV|dnfP=kLTz}6goe>$R~7n6)IT&BbwTT46qZQcOF z3S+Cs(h)Q_4PED}MQqx_4N>2<1**^N=aX1(5k|jWX?HW$Lg#K-ctP08QslW?eu9`an71x01>dn(Z@He_BLHk991@Xhd>05mszjPZZvW8>89oY+5}M)1%4_)5eM z*2-WHw1(dQG?S*o$!H;@0>oy8&4u=Ta<2ijLh{mrG<5XnQ9;332n%a~!k15PYiMkI z2^=qANk&1vl&-V?_T?NXmJ>IhMn)bg?|hG$B(UHRJmrdk6wZwyKrVUa`pWyEs56lcdH#q4DY3xI=py{Jh1Ls4FEbZ)$<%wGM*iAmf zFO1XR9>jUb8XLGRe*l(I-MBmZ+BzvWDYDwzogA&%GNc-s)#Hgy0L>0G^u1MvdS&E8 zmR7iXBwVHtkTO8+2L9Y)s)||ouJ||d7j@|euAS${FmKv+h zl-+JhQCDT&IDrh1a(V-tXG>Dg_vYXF{65mEXc7RBkj9I%P#Q1_8B9yOM_+q*mx!d@ z11v!9kia^bju=(ocNxPrEBt+%!9hq3W&-{EWDa_9I;RCd8l#*mpE8#|!Keq1jC)`E zh+7Un^!DDQiq6@@q$D-t+K2n5GG#Y13I^gY?04?Tx7+}N$`57v355Pha-o)_898^i)j~AZ+2$`kQR4D#}*xGBTzUuy0eoe z1T@n+fhM8!J#1L!+8TR0kmY#%`FfQy<3@v>rAF(vEzwdJ<9C*_x3!2wf{hK$4vH{1 z^1R4$h zyvnu$2t{q6SU+r>WK#6!%%k%FFwsqPr#GwN@wENRTT#5j0w^jGOWJ zOT$?oFJ|gpMkx`}_$KfP`q^(^8zXTT?!XcN!&z^>JOkp#4J-ihhW%Q3#x=X9f`ft> z4AZKha(=h(eot0@lYx(89Oa0G_8y@sv}+;E<^}Y4(Fw}9Fho((cl|sSpe#v9NU$!Nd;$1OeH^yA0hrGsjuEnU zgJAK^wS}=x0zYLw3dW?fGvKVtywR2UAx$&j!Go6|hXGk`K7N(<&Zp;Mx6LT2CVC69 z)|ESjeT42MzIY)|hx5KM-!;JQG`hR3Mdn|Qc@A2`R&69+#5sxPI1pzPZeF{+Vt#$0 zpUL%G(t_79!0bD$lk52E0-WWgXJr5OaTCntTr;&O@jJlZmcmq)nr{EQ!}wRuYie>w zbN2m@fzPAhzYp|>DpjFOVW+k}Mg$?s^S2OntLhw+-u7RDl;f;FnrM5a>csC^^Us{r zFI0}TwR0>4OTe-q$1&FR9j71u%iq9TxfUB5JQ%KBt$`SPh5sb`kcRJ>Z>Yn*sl??xKQa*J5WP>i=$t@1ThI;1)En{73Jk!oItjJ zaK|ljl&*3$NTG?$T7LN=^a(6E3K09Bd#%pC=#|M@Sy!=Oe2NH=F9cWZjM@}RO_WbN zvk%CHv2X%mrj_x`On0{1wt}F;l; zclH>-o+$9c7zaxo5C?CnJxC~D#>Rkky$N>To&zSJK7i}?oT4JrQIpyb9u<&R1yrV9 zLHP@{)x#ALz%FYEW>YZEbmXFZ9j}yBl)9Fwsj0HE{{$tLhM-s0t=EZu8DY4jx$U11 zS0)8AQMbSf3gZhdz)r}$^@=6#(Ht$sw(`lH%uy1?ULt;sg>mq5j={ z4ToLxsq(^NUAwb(9g%c~bfX$A$=8#+D5zf~U|5EQrLaw}puPpP6R`4i!m~AcecRLj zwg7_DWuuFzGrG&p*|1tVaOV*v5s2%MuW!Jza=L(bfcBOefQLR45aBzaalxS5m`ThF zAS@3rZ$Q2{z&MA-r@~={Q=O^CPtQ^Y8x+}@@75+Kr&R&SmqJHNn{%aVXB#tSAHgCb z_I~r*QTZQs@G-EojBOaq6)92?sY=evIy)ClE1f;t*wPZNX$~4Wp>g8Xl`50qqEH~W zo+I52P)^_x02aDU%eRf4-DUbUcmQkwKKazWLr=g+lN1wHM#sjQpv55CXvp`Z?dW|h z1=Wx3^S5Dv3JK<5tAUa?(@vOSUqHsPN(Z>P6JV#tYv7tyK`{WQv!FOPH#czN0zK4H zOBbL40L*O)ug5Fy5&`+Ea_-zRo4lVH>?UA+=Q@&uCF8{_&=cA~!vZ%dTMPh~f+PWfE!r_g@6@c)6LGp1_4Efo{>@X>N`?p<+3UUK|RN6dhPuO zfFeJs#;(R@mAZ36 znEzpja>L0cYg6f2nWZZ_rCn-WdbgW;@@^~-jJyIPCY&NFmSc*joCX>~MF%~AtiPun zLgA^}mS@&>707qzEE%&FU&VMcXKXERo2A%=9PJra= z|Mmy!!d@Kc5Q2}m{F~=Pjw)Rx%VC=r)ju%;^fH*X;@HtkrOtO?!+iaa&KsiY>MB_s z4NCm51QH?nPk#uos$6yfq|_7o@1UIn_-23PalwC`7$Q{MsLX4H4XxlUHskNM8q{_# z{pQx}raG1WsNZY$K8902c@sz0=;FWmo&sVpk$)#h{yDAu;09nQ+UUPwcz!Lk z|KCo<2fps^Zs+hfgbXMrx4G8h{*l6)iwif`v5!^oic`=NFgc$I{v$lBX+{xZ=k}~2 zV4&_O1IkzyybawcV9SC|3&W)|`?`EM8gA$Ag)-Io4>Z)AHAP73jE=-`Hq$h264Rqz z-KYv4`}%q>BF^`pfTUQtVz%qxSN$P+Z1uG?2zKSb!h-cJ@`Pm(YSXz*=@15J1OunB z$DPtn3noxqT{Sia$nHGV$tP?CX9_B7ke>a5gWbPo6udj&$z)XcIB}OWsFPsVxeNj& zq_w>g_GfpsvXoT;Vu5ZsX^WfiM?PSi^XY*{DJv=Mj#RYewTNVa3Xf3Wg9{Q5nZIFM z`$%;FglR|z1X(;o?@e~a_}+jMWM#$PMn70SNn-MsX@7BFKauaz59OOP)&#%z+T!^2 z#;2#(6ztj;q~L$NyuLla2J{1{0q$Ly2(pf_a4Z~FXqIbiXn>}SC)m?MZze>0m3c_0 zS&mH*7=+tjC};SOAJ z040(La$T-#^3H2<5SR>(X2x%*UK>25k#uuv~tTX)uN{x(f{h z^56jR`kZLL>Yv=pPs;1!_IW6X#>34^wB%$XaU1 zVBZke^DY$fD+{oR+r=_+e8(cGvuS3bMuobh{NICWqeg5++TU@w8t2ZRZ*FMl5dT2$ z3Pc4sq1CZjpyK0R*RXG(tEi#r>gxBw@$&HaSx-KfbOO5XeELGwLAeL4qO+h9y;uRC z73LkS>is#q+35M8y<+jRYXA|5y_tOuoZj`Zbrj%}%=uX7pu1;1^I=dwpnZAo-*Rm) z#LZe`!G#)ybjLwwjOE9d5fc}+(ExeSFV01posyTAhl3}3{pQVSknVywkOSZ+7CQ;^ z3{+vG7XOA3|v>H zFFR{aMNjsrMv2Rbj_s52(JQKUKW^3M2!?p@oup%EiQ_e-YU0m%4qI(Qc=|h$(_Pa1 z$=?1I_apc-`k^e)Dsx=}G>-#nX?K(bMhuQPyu&#no$nCljm>c7q0B-qVPOJxwY34g z=pgEdZffV;d&on42M&D6@3l(EhheL&#BKxnlPV7^wC60zS>qFRa?N&EKLET7AnjcL zVuMim*{p{K<4}(X^^cT)e0jMeUM_$j7)|dN|8QG?aE7#^Y!HYVVb#0a)Bj@m0XU3e zZEF_XvK;Pe6G(O+-cc%`q5dXXmi`qjEgT`Yc%a|@XMI14?P>V+QC>0VyZ;}*PYCXJ#pY;j$9uRT!2dSCb|%(aENzMv zfjIREJs@(G&{J3Rmb{#NBt8GfQ69b^c^~?(5a@X*<)*&wJcQ-H|Cj9J%U#(5;7sh} zZRoaCO+EpHg?5w%bU6z(sc8-(IB)hrFP1ao4k|$5B})$`dh!lc2JgesK?tCtL7n8n zUbuMjix~Kgk~B6z6I!~x3-PB!YYCW<-it#9%Lg%;l@n{@PlI|_P`PZd3alzA&o3oW z5mjX{^UtS~B129JotYt~VA#*Y^O#TL%WUzS7U-p2ulQH_Smn?Fga;fy2gbA2qAp%V z22GR4A{ypffdLI2);KkiZgmfB?1O|mBW#GqyX*;ArJFcge4~2@Z%>?km|L;f8@&Xj{8xRNW1uh}btPAH{a{IhYD6c)Edy3V*q&xW7I<^X zW0)7PjZ*9kMbg zC+PDhD2mXt0Y!9M8=)`qu`vOA#mSJoArbF}Iuk7aT>Sj}e0+Q&A{`C|wl;5*TwYA7 zubwr0|NebGTrUB#LcD^NAZz0nXl_}$x@XTGI6x{Wi}n%esG(CxWFP;1O%Mu=tKi6tf+wpPoBfmRXWBb)um2e-LK=e&r=D5?7FFpvJMKwwvTz!+tfyL%dS>^uZis^_m4G>vV+LBZlEp@w1PsG4w5MX%c7A)Jj9^S)x;wA=+tEp{8cWzi} zLp5^}wXH(Cf9G%wr8_yuoI9JdW(e6iJb;)9MWEMaplNXS)9ZUIFh8#4pPq=zNh&HT z(rh;KpIP8LLdKMM#+$rTTH`(o_=>`bQWUHVE;vr+WM>l^NWqU-Cxtn&-?YWa$q8mY zNgf!LJj-j5-B|3}A_%z;xq-UWA}C}7zz&B6Ktp}n4BVi!y+#OuN$cgrJ{?wtOD6g% z*7COMGuzjQD%Fb@>GE!qt0h274-80ag0ilzoOPh^T zZIU2O;h+v_*j7g6*S$!vpX+fzC)b``DxfSe_NaB$eX6|j_0KHP?oj%g-zZ%jBSKMGa7 zt8|kSK%$ZCuxdEy)d4Kff7G>8(QeY%*flVz&*Ok zUNNKdQ=Ing)|)oW2X0v<5ZYzTK)>ybO-}$Ewf6^G^UpKW0^S|2u)&$QoiM5y*iD7$ zD|3T}OA=fUM>UN0u+{>m+gO_9#t)q~Fo-^JDqaN-rZ6?!WOq*T-l5)yol?|n;urgG zTUhYVBtw;U`SRt?h)0ihGcqzVGb^bn%zhi3IWG`sqJfDta;MFp3hhC7RQxcAFe3i( z^h!v$fE9Wqs|a-1DInos>Fl9PCpWu%?Mvij4*xn4qtOB%T8NAQ9oOq(!iU2yuv17< z|Dv({U4Vhmwtb&gud{W1fix2#pVV-^jlgtlV>9>a^*E$rS-w{j$KayN8os(0=Z^o1V zv(F4!V(76y6)Y=_(Xnm60hPTC#l1T&95ZV@HK&f&oSW=>vj`r7B?n5cLqBrJb`Mz4 zv4slCAAh*z(y3|A`vvr-*Q63*j?t|Ttz3JIEW4sXC8*LmVJ&yoxj<}CdO79EB!$v zJ1ioCnVy~yAWDZgbZbJeF}KSs>d>Wku|$X~0slNU5xt@HEC}7s;ks)PJ8nj)nV5_M z3DWXhEP)Yg{Pj`@UU$u@-n4N51~yFgqbT`fS>U#kGnnB^1O1H*g(z|xCpKkN*+rx; zS^mxq{+=&5>oL5d%;!deE5f1gR?S>FKC?WgqWIs}B2a`J3Hb}sO6G;Q?VB6GHU*sT zX#siw?o0{?kz$!-*IpVa&%G7a(b3Tc1y=g={d?Mg!cj6L2wD)a^x;OSiHmh^Izd># zSUS)|4L2@uaNyswT2EZNrrju=C!)R<9o9WW8sk^LYP<+np8!t-$$S71I{5h?7q=fc z*QqL`Oxf0zQmTua9uUko9d-n_kLNh7uZwiUwZ z2G!Gc|ISq}&G6?%6Qz5-l9r;)l0T#d}*PBcJb%v~QI)VY^qM!9^4SHerj_@+r~1@2pok zSfcQPUfo~HxXt6Bi)AAvCFSYUr=Ww@ZwtW_lp;wWJYWewX6pCaw9T=SQEm;2gO`W- zeAw8RW1J&g&PruOu7>43CnVjf!pG_QysM~wOPH4gsR2`d_anZh0Fxe^Pg5q;9$MP! z0zPPU`8+Zr%R6%&zJAU7>vUq`{%)sR3HZFG6RT`|&&BWXgNpUO+Bzf!%h3%Z+0894 z0GDb&kn;&Z}dWp&~TEm8Mr-x0|R_^|% zp+F-WulwmFAZFr2S46;loS+cgegH05*%Jj!G+2DXi7-uaGM1tiodkkc%whI9rkBXp zV-#Nt_X5FeeHk5Hn(hBFL#fe^>K79CSNR?7)05JIQbiOR8vzaj03ukWXO)k2<_|w- zjV>E*xJ^D=iJ-CiGek++!_^yv7(iLo0xIxq;pZS&7-;@@+(Q^6xN*O^nt4P;we9TZ z2BgHrb#@JjGE(K0LI47r7u!;Q`~~MM9tY&bxoFAD{q?pF2)7Qg zVBeB()Uq$%56Si>Iw~TP5FZcx^cPHYKjOdpehOr^8{CEjjaDzfPf?ZxH!TUl@--b< zlSeMe`7%kn7HYNUUkg&?I|#HQD8-=i_ktH6E`)h;J}S+!`oV5-hgmE%`}RAHM_D8f zZFoj;ZI8xqL_1T@Ru1pfmfRv1{@n6gZeYUcV@54dbSOeY2&GEcm0bMSAnPY!%C0KM zEej2X6dk_(4uYU0bTDKCHcEqDeVyDftwyG7Xub!<4lELCr{eRc8C?xuX?PY~7p>4g z**Ek3yV+UZ&|5NOMI@_fYz>M0zQSS_dv z7~3A|3L7heK%EdQtsDqdr}~EbJ@U0Oem-8_aVU<1f`Y8t6YHQHkiBkiZx0$TUu0Xk zoqc?$eaN@ewf5O{H=|!ZlRenhS*;592?|;*jMimqV1={@%F0Y$SLUVHXY7hN{_Ox_ z&~P7D0l47d_Ansu4Rn;+opXht8->u{!Th9?{mXqY|4TS8rI_6qg<0ykqORT!?QN;0 zV9&SE2o|qc)l`CagGd^JH^IPljUB*t6K+9Ei~_5idh&2Vj>$cH zMQzZpxv3@59@B~Q1nZ~vGQ9Q2bZyoDV#JM=!Qf($5tEgtcw-Wt%RbWpw+5_2Kg0=v zRH??@2?+^czE~@_q_e$k+6ZD3$OF!6JbB>)m*YFB3?9>QaJWM*%E*CoeuPzei5zmg z9Pi91l*ieYAE5-Tt*eXMNi5mz)Nr`KZnkd?E?t?IdJ;J-NQoHs`2!ujHD&}>FKDd4 zNdVhg4*hI!ISmTDGNaTEbs@@M)ygFuX7wOlcE(VJ@?^k1fTl#?ez?jLABtEHNVYwR z6fp+7DcvnTWfohLvbOXPE^h7*95J-u^x!UYkj1Z~LKQ4GPMDQ^S^=+8_9FOWp=~+X z-#@;yz%`@+F`(1EZPxL3*7cmnK>^~yGpCinnM2-f# zW4Vdx2ipNr!V+Fnxu{WcIgmKObLyOzp_TezT}I>k z){Mn!MngK|L~s!i?uA=WIe?|yd(vjIJt-^-l0{dpxkg(zw4AR=AbrZo$0D6f2$wt1 zwI4j*cHZPVtjGXF8aM-M2b4ss2*=&ucT29`qFa82)?3s&A#Y*HkZ>KG*{4@=n7}Yn z;*8F`QSP0!Z~>w}MujNbBtN4_6PKj0KkRyD@vJzx%JS5QNU_^5_P($hlQL!730s9D{OAFYefM$Z8d8>vEtXKsD&WG~kl8b1&S1jTF z%|iGMa0*~kwSrfa8;52&=-NAP$kLpAV_0ynl}+_G5&Vx}^|Ofb8BR_#vJGfZvjAuB zvjN@sq5ta7NRe?D!KVT8Vzi{w0*G#3d-Bo^1l9g4d{(fB4W;}X?BuYW!T@l;h_nLV zb=LPk|8MdoH67Kzg)2M_E!QP|DhRb9tV{Hid8Lzqw}rxfW;k;aI*t z(7EhvVL?!W23_W)bb*L0`#u%odPP>6ME3&2Y44QND_2s zdXSb|098j1#3yH4?IocmrqUB_3BrB;?%k%AIdzE%9AdU!&5A+CBva}ah?xwyKLDWe zM~7M$Quwm!&2JGUf5OBcbVTG8OzNfASFNlv&CxvtmiIUe&o z&&MH>u|jiGlTo?-vt$Ydy|m|r`%E5%!N>uD<=j^rA%)$5tsexTBVP4zGV@UU%ZiKF ze6L=;dhOaKH)PT6BI<{Gr_jZCxYX`OAHP#)`M-T$0q75DDt*>{tm8BF(7M&DizNif zGwAQXiPc7a=Hv=Zl>|X?8~V^Lj_=)Qf+HRI4%3H{-(kLn^U#cWqm1T0a%9I{_6>Wj1%xi5DuNIKA6Dg|f|ASNK>z0V$2Gu~tcpD_wb>9m4ravL$cz0i zB+9)juDH43-Wn?^DiRVuW1O2aiSL%l2|!h0{MZM5O`qPU(@nIFzsP4oRzVYmf(}u` zr%%uwcDIiZEMVPl<)(J(J`{2dsq6ZWX&mIU05uY+pjZcN+zge*0v{23AbI-+-KMF7 z?EynWJT(e5kPN%6t=#=C_4I??;ve+Wp(fJ}Sx)E`;q0wEkTtFR(a#)jAYY`N&N-#Cvsbvw$8~9J|ewjOXxYd zMP1o+Qihr{W_U9xvB*r@@K+=1nfH#R&p`S>}ltGgi#|KCCLIV`=!wguEA zwbE+}NwT*&xlUcq)*LFxpt$xOb6HEvq@0+^F-gmO=a+6SwhueC9%i+M)Wj>yeoD_M zmBRM4XkAO{$!9!1U+&NMIXW~~C?1SgGJ4Qu`<@w8tpr6nK7MtY%@HT%#jvuKH>VV4 z$$1&K&X}a3&GcAi+YC!4jr%JpN0Xa8vsxX`7n(fFDnjzp*A-*0Z5k$O0Bjn>c$?qF z)e!;#Wf_C&B0+2iy>!nsi>HZC_rLI9j{P)}V(NR7tO;L>hn6I_K)=T5DT6yxOQPRz zKfPAy>hMp~k>=0&FF!{Lo3yI5ap&aZo5pKt^*OFs`CBD@JpbDg8k{4Ru)XPeKx=ia zphp#XDcP*{qXm=B^y8)C9!WGKOWnGO*rwAK1&UV3&AgqQC>J#*V<+pICx7RS-g|)j zdL@N;5xKyR@l1=gw0O9=Giua{Wj=l8Y>IQ#?ge&YbfnOg6fO__q^9#UtQyWv`KDIP zPt?ogDi>Fq)J8I2Zr>iWmyPE%CpGB6V zYm}T_-D3{EN@Iy%U(lar+X1pHcacf?EPCN+#$LB?Ce%g$Wu=bquA5x9I=2I9L4$qFAop4@#!d+F%*{PFiWoB;v zPR}B*mfo!a8@vQYPM7BNy;+gb&Z#m_K4z;grz5dn&!)Xz z-h81Ejhh_X`TxmZvOn0lb0<#VX~+M`U(M*)VPf(fp~hTPPK_(XE^@ZdDEmk~lJ}+v z9T<1Iaf*sF-05t;cky1@JbX`f&H2REyL=ABKl)yX7Gc9B{Y!(Y z$HbJ;%FUng_iNxK02(L>bwL6ji1e`Jn4H(bzuz!^M<>b~fLXJ$vVgkTf$A8zoPBI^ zUX%Xer}oGVvPoODPG{9Rt`h~-#CBIMZn}T?8?!@$Y)Iu^kfxBiok$Lxf4XC1jfjTv zJZBs66ZPBL2toq@Qg_Q3zYNjd?yM2c+ewbg`*rol-eI=SFz4oGW^q=r1f$cSI8*^V zCp574CK~xtw2za|<*6Gk{eI#D7@p(o+%LD+YtNx}NGsQ)%nHD)rm}JzG&QPc{p&xM zg|PNZM9Iv|%+oU(UsuyC)`)ORE-+)VYcd;WfXACEKVdO@lA*QdfU?FlTn1h73F9Nu8jFPph7L)I^sMW-c(3UOJlb^ej^MUu`SffBt0; z8SHD$`Td)xQ{gc4spF-*tNYb3jCCVL|8`M5^joL8N0n^((2)G=OV=c980-J?MgQ#v z*AvTQ=n@$j*?C8dp(C(z{@EzuY7yz&C z*zoYTZ{O;id?}6r23io&b77k^XHG*y!}RIX4<9~^)K}>KEV;fc*L}{pK|_?!GJ6L; zNo1S`W`>rQ_Tg#G;D*!&U*1Rn#(Sj&ca2v>p(_5Cuh|oLT zJDZozbMF_Y%`yKCABcJ6 zp>hl5&ggQ}Nq7HZu5o?7>OS2VbSCZDfzhBv+L6<0myob`P~V+&x;<2C zZeqbyGx1)QTBR=YUC4K^B?=ssWs8Ut{o4yt{n4Ke&Mkuc-`Hqq77G?Uv#5)!>*3Y2 zwzF)!e2&jbOGSk{qu1c*(Jw%<7|k~&cGTGsx$%}Wx|YXr8Gm~pL`CvL>@0mBzU`dr zIvSDTS~xa30=t7zJ(}5`NH`;JtOCn|0Eq*#g7lJYoy9#PpzJY$p{h#v^vzYR)7*2s zON%HU&{~c+`}D#WpN+M0g;rX1&tdZIqfhjz7IgM8LH^F}c=K#^T(uBpUS7J#Yj?HdeopD;;BM@0CpJh0KuZjg{FiZhZ%+YqD! zH7%`F2)di!X|rXs;}4IFGz9Rfou+iPFoCsJ6msqs(Q3D~)ebUp+LRl~woD>&>-niWKgdza6IW&W4+6x13jdy&!#l$c{Jw zSX~F*X*GF^|CSLDm+R5GW$l44HiB<5h>}ntZ9nG)!bD9N*;Bm_;V?4G+aXtlhfKN? z*g2Q)<0wnS}wDRl0`|*W1`GIIuqxhZR^{U z(x^`Rm)H6w51`uO?Z(;KiZANzKl4$Y_FtX*Pd}6$F?^nKW30vgA)ujpYZZ<**2TX$ zuV1qv>WlxE?mu#uu5JcKGet%Urfu+o`u#t~72zh(fvG&xvItd2G~>J2m3gPS+bU>| z3wg%poP^I+y7#uMOho%}XRxcV;EEMZrBSTcl=ijaodw$B=(ENJe7~Q_^_t$gU0&C( zqoAXd;7anfsc=eJC=oVDn%%ScXA=5LvWIm|4UdF465WDbT3R+b%Xxzmsu5q_9twHo zw~*y$di#f650-+e3Y*n@hI6P3uh@OmY(<&D;;rk1MMNYLJIGCogIK#svkCVynrAQN z+!)Mqk#n5;v)B11K`O#T2-rnjObty8z;7yzSkSe$-eN*o{k$sh6UnxNq|-NQDv6(G=y}EO(jcfBKW{^*q1FGQU0s z5yVk0(3t$IFT};X@>{T}NIR}pL0BfOur;;+oLBs~hW{I3>A!xXe^{bMHl7*E>sH41 z(QjuLIc$ERz;Yy0_|H*dm|MHAAY4+STMJNEVU0aqg~B4P^haRU9qBOiFRPn@}V zHiv`>Se!<7c6J92)X<8g-YQh)Oi4`azIiMf$lcFiXFcgR^ihW`Xa!N*W$LQCu65Vz zTf!Gjb7!i$Qal|&Hw--y*8Sm-+HNhi7$0$;UEl!ai8>^&2iGDnU0j-M)mBRcUIf&G zm`4b-0E)&?2)`m-18YeT>Hv+%mOVJ>23m%IwAer%w~f)mKEcrSrMRPm8z$pL-@9jI zXh;OQB^mo@?Tp&kFh)S&b&P=Oop4t6sxmTAWiJ_h7-?WqF0VC_AG}f_+xm`d;l=W| zoF*qW!f^Z{AhrM~HtxBNjg2Ln-AgV#J;bK1Q^Vc0wY0Q!b*tKQqS&g+tjZ-StwQeI+p*Bg$7iU_en2BO zG%dtpp7fgZMPbv2Q(c(sUk?~e?<%9n3jWSVWrRr{E>bej+jJbwDKoCbuV~ywwjsJ} zC)&i#ipQ$b(~QvVV_@5`(5Z}f#w4>vT_<=J(5qw|*%cd?m}r5H0Jkqh(9n=E30R&GZqH|2K23`rI#$QEHS-JoEpv1^pjx`+*PWlzYwP;% zLr-I=f_Nko5n}VsVsrz~!FTPuMS`SrVZ z{6~a)p9;k*2dw-(CWqqFC#bxwUU?NE8Bhz^AS#M-B=DKxD>FTPy6@b zKx**w^Fvle!Pa&F0;$)?lh4yYU^|aAXtrt7QwV0Nr?lBR`poGoFS6s+;SW?Hp1qbE zVXLGKIo67nGI}Zpv!?#G?eJC$)*Lq;Z&+qKEwRFUPa=>9lbeBAEkx>ebhM&Z_AE5j z1~GQxa2=pHkZrpIAuzB?d3h&$%zIu?kfQ+Ihb{u>m5YY2jEIQR;=6V)Z&fo3E?+K^ zTs_6C9WV?g(guZZtb4|%4k6T6@FcD1#Cs_ZGH29*0KRz;*oAMH92vsyW~$%BUY<3v zbxrXfZ(Xmy-4H(ai!zQ9Z6D1A+VIPYiBo?m^f7czpdP zuiGj^K~uvP{Zkoed1PZ$V22_Hrb}LMQ@6D+i8dEzyhb-CXN$LVjrVi=;GpnBk3|z?9`WCpE%nW|MS&^1wh{2rL3~Bs6d|dHJjQ;1#GCF^P`L4l{|ae#9^94Bn8o2?^&mOz5UEb z{}!PoCfG3~jKuVEa=!F`FW$<@**%S#^oQ`RzG?UF-QCl|Zb3-Ml#eg6kJibF{}?%8 zi<8^rr|t~jr#z*9bhd`(bg(#O9(1k@fD^6}Mp{aXYAwnN>*WP7*XsScXU>s>Q-bfA z0ix2l^HgrDNy@Vf@r_1|owl16q|#>nY$EU*4>z*Lv~o;G`b*T%{=fTx9j0J>8-5sS z18UCLMDtSd45A6d;8E2ocs8nXT%%+75f~VH=ManfAO2ad+nBO>+PSffw_O$0L{BZl z?4ujC5jpmusL`%}`vMPz?-@gbC{KNJ7dX87N6)NsQgx)nxNP{YQ}Yo~4^cZZ#>du? z2A6iYD~+u7)G@C*{H#TfFhhYN~jsU>ZNad3L5ojl1!^XkeGn1SSe9x4o^M+aBsGV@#dQlFgBI5movj z*)*QLmz-8F`oq&Q_T1df-DHActG6ZgxL8~$(LLOx26*7%!wmp#p7Z4PLAv_y4<5jfSR02<|LN^c&UzO0Kk}#bp!h8*_OEe70Z_|=jTsuxV{gu z+@X+)kL*#UrKOPjfU*P!B#1ydukZDgUb0OLGA9WD%s`L+!n?ZXxk%iA8N#Tb&6G#> zLw2Y4w$9G&VJ?LtOb>EwJMC*^LGxD31#o8*<;4)w*w|J;bIKyOw>CI)$_L(Bv@DQ! zebL<@JHfdG0927ct!B3ov%8Vr7WnXI($7mhQ9Cee`@)CSc_wSjcdyH|5@n|JHf&Cy zb#c_C)Ux8NYD{HljH`!BBNVIg%#HM`9*uNfiFWtM^`e&Wiv-+#v)tcBuE%sU^!u;| zo)tHX(7%I`*?MImcm~g)Om;Y=;uCvR4|E@5?7Gm#q<7hZ5ZmXdT)$;{N5IXS>}?sv z2&eONQF~mWl^mO%Zio6BaJ?G(ks54O8vZL!e%jF%N#M*ndD_$)tpLR*B+HGA=9g&< z=@gJ>sV^PeR*iY;cVsZ=@#X#gGYm6_nyUtPE+m2EQ@f*=F^8<|*29;-)-Y4yg2h|v zTQjwsm5vS%$8^RG$GIvTM6Gyh!*i(Z>amA?_|_eyBM~BGs1}3FIw#Ip-3@s^%lm_7 zvEYV+_VIkn!#Co)QM7l|xO(|@@S}$V0GsNa{AS5ptMe?kxLBq4p}e-XHu_^y9i6T( zT9zyBs&ZTQ-YZJ)E8t1FqRgdEB*wMI0CBeW@IXey1T{yfXt}M$+%^i3OAQM1>$Zwl z265Gev-Nx4dywlFQC(!3dTXsB<-|RqwT}ASSN4tq5`NjC+}5St=@6DV4CqBiM~8&T z2Y!xqsCIuBQ}%Q-+ty9=JW zm%MwIeCU~Hhob$nU21Bl08kMLZ0}igKW;B-=)%F`yzi&Eb|ZopOZxsghxV`@7Ms#| zD={$Q9(Ijz7ME*2Lluc*y^WQk;*}2<3J7C`p(jwBG=PF7r){T)k=nYw zdg@IN3Z6fgs{`;DEu$42rYX&9;ta0m$=0fn2EHogJgb#Lvmo9K&@n=^qo^nY(-U*z z#=aFXD;Nc9J7i8kZ81*Wan(@FE;PH0U5TlDlYcA(rpzdSk-O+ItT`mGZP5BGSWSaX z`XpK%$+p31Y+Hri9n;g_WO`&CnUYJ(P7{p#!KBPfV2Vav!11rkF4TOUuSb?#UK}sk zy`V)&^eH{VVcyiqPt-@l8y`i0DB$Ow%}*u0q$Bw{j|2l_eP!|~V4Y6ReyVh*dcfca z`Pq(z52|KnZFIP^qUyxMRzFe1f%D)VgA1%vKO9-p#awBoZf zZ~K`V;RTzmRTf5Tvo=q^VR4V zHn_7kcgQjIrl~Y3ybl}Pa@%&6%o6s(YG0=i7Aj6)>^oA!{gW=MC|=&)5u|JKwjm2k zx;4Vu%WtaMh0Iq{aLsRF?!AzxVjuEKbd!?PMZ%Q2SLgj^ON9ea1@wn4W$)#D*#sC^ zn2*k5OkphNw@At$=ZIh;z3_H#Z}0dxl<83<0FIBr{yU&60gv`)I)J#Aux@8!+1s}+ zbgH{{9S3=$q_p%|Zgx+x&UD<;9`GxfThdQ`RWVPOnwi$#-;zN(^5O=CLP3v^5M)lX zX{*gaDTUVJ!Q;o)AhY0B5D56#1CB;w7F%Ce7FdNiT((`6zWHFr&8cZ7mVKZ2&OB;! zb6|{#>EdZiH{e@PC4n{i^PT7MChRPsL#?X%2H=O1414=>{=`He5s-*o9ono|z81F~ zcp#$Yj_D5|v^DN%Ye#1|U$311#cxhiFTO`5W(rYc#?V%AF)^p9U5w`QtQ%w^SXd$xJd!!7swwCPnNFE#zUM(ZaIT@#xS zE}MZ*^*uWS_S;k)o#<+Ksbv*G+=2vACX|pi$t^0y9%Z&z)XN$eL>S$Ridrr#yt@?P z!ZH2Xge&AYh{Dh(_y=#dXO#+zrlME{_QE_Gd+duPgXFn|xh&ay~ zr=Nx#7j>|ppr8!1x-I1kK75RbNgA8$Av-Deh`j$>m@^)a2SGbUdSCx=DIl zL$1W^tTI!@nsl~**njagYlQd2%XMxX6i-f3&&$clX{U9osHgyl*Fp3`a4`3yX2ofh zwDVZ!4=ttkyCuG7`PF!XO4knA{1~8W5jW-fMAdnEGMX(a6+ZWUTc!e=soK>zk!m5& zuH~Hc2XC=&7)FN70G!1Gx#^#RtnUBBvZ$SUR!;PWZ=E38Nkn>3SINo{sY^LR%=&4U%? zc_{NF{~ksnd1E{O*Fbah6BFCd;g76kIL!S=Z*`k4i_PC@HN&_JYB-2mN!c}{W+MN- zz>eQ>!b^vdQL=>R8i7`hO9Y1R<&b-^EL}8#$2qjw-r?EliTohvc8=dLq@%ebwJ7u7 zePHBqffyZijno$XCIiPYnAuT}PwjDftr-K!{V)HFk~?x&#}*5y+4MgYQC9O(5{`u> z%}`TU|GHCmzV*g0`w~KrE4FHv_&U?H*(_LE$G2mVmFi#DwL>^jh(O;6h(BBNVLN2@ zI`4?gmbuMy!E%;NCRx}S3Gf|x8*kpuX(4Kr>r~p{HaiPtt+d@diYGBV9?aMYUSWG z5Ah5#<06?J|2_?!OkyzMc&L?f^RvXR?XA=5R+;ROo(I0qAViK_9YK~g1amp^W8>2W zDbk3tk^T0fPZI3MJ+v%+`yvfBHEFyeFj$Nt&`G)zxxV$u)L=7YkhOt)>Dwnd-aq#Z zYYtlP0T@pR8NCHT%8ifz<;$13smgoz3VOP)4UdlgUKSH%D50|~|HeXxyyR7F8#ED= z5@?-FG}1QTRZA(or0k$AEGRg;lKUBX79r4+(Q061v_e2YxfGzSWBP`9vJ+gUgOxdn zn+a@M7;kb7i*o~@qJ~f~02~P95k{r(`}fhEvxAApKf0X`p*EVB%7zy!o^HR5Pj+s$ zj2`t_ zrpnt?+xOkIe~hY+gWGP-+;ODd?c;o`{2M4h4D3?3K@)cEaN3mTkVS=F-uX(EJ4 zaiK8}&Jz%w=egJG-&^xoIV9gAPePB7MME@Q;b6(CKwjy-FCoDVm^r2_By^p4LR898 z*3drHK?LGZ1w9OY5QzwbM z)28*Dk(gZ;deXC)cx+9mYA=pnDr7+ABW!N~T;u4k33bS4U#|7aBV674z21Y>M^ zJW>y^nGX$VEh)daF3q8(9a}{sXn&G-eJ&zV40Q3}q*T-F?cdY8GGbeIy0aq(Iz3|v z5u?Y0Xvz*Rw?jk;jIV;(k~9R@CezXgjJS%2z^~YlyG_Gn#}uS@1W3*tcooM>3E4TlA z=>yG-B%*;QF+t=|yNm3ByN9E$tgar$nt%GQ4zBP5=X0&Uegbh7HQJca&;AcSFoB0c z8UO!pt&@N3t&Ds?GWmNt zwfF3aDBH2X?!&o>o(oIyn1kIen{YvgtoWBNf1~73DAS|W7KyZxm;!^Z10*&5jUhnP z)-TW3MYeX6-aeaDol{>mQ1W5<|uAyczwk3jT_LK}=0J-+$mRvP7P-Tzuvy&s|z?ZGnJV4<0;sutwgK^oLQZ=BskN~Kp zA41K$HU^E~k6^a0#ps~(Ml^@WOSprZLbq}<*mYiTaB_-2yJI>RvW0;QG$}s zSo>Sv)z7C|Dzz!*XdVPFo13kFt$ETHUIvEKO783FNOE%JvqGo%%5=WnO#tWBAuh8u zu^_|c10&^FD^KbD>`aaHcJEg{%hs`&^67Fq0 zV0|nKH4>L%%ss#J@R-P~!H4MnH&3ae_}dTmI)6TDztiFa)a1-k(JVv3%*heMPVz0z z4mG(s@%w#!eJP{$Z1GH@Kdk&h1-QPOEp|OA&3o<7eg=4ei3a=y&*lmR2m1h-Gl@3N z6kkrWt5#RK%MRtSYM=60OQ2YWFrv_DYOf#TN+4q_t4Hq`P@Q77cAN2&Bf zsUcuh|LQKXvD;y8CBX~8vACz5vOC6iB_(<4*j~EXamG6jMm~LO;6p0E^>}p zDo|4ADrTKZ99^mmnCI1$T`&H2cd@>O0O*eYnz2^R#g~pmq&_`qyi)nCbu%Y6j@N4V z6!6?sCf%4oW%Q|)Ul1BBr}u0+wY!CsGk5IkQ4!t$>?S>`7UC={uKU5~9j#hWfz@Gi zh@ifm;|E=F?AH&oo&FzwV1jV{0Qq)8Cc(IstCyH?9fyG@W25~4uYYD&H&VR;;I+ zm{ul(`87f|AM>~_AvIg;&qJbH8kLIn7`WwEPzOO-7ZW1dkz$8;VBV{iw)P?$RY ztv^RhQ$uC{&K=%1i$>j)=}If@+K2ixhC14H)6ZoZ&x-mJj_AW|Qn3%p*l&U0C-7xJ27(@{rJuOu+S4Qb)m0n(63yQq*Y8`5m0f>B9;7EWQP@ZZhL(}^P zs!1^2=H%7`<&4sz;g25ec}PP%22goK;1N(aXTo(lz*$9wx1PGWN^KkFn5+%BT=CGf zN@tBr9q<^Ddgh-163ISCR<-pF3GgLHR-2(CdwIFgeQFhjvE>H3B8LlDns?kD#@+&J zgT{<{b0*W}Ab^8YU8V9kh7bPlcEBt2^wN0hnG(9eCWZzENRz1ec=9}8Fv$I=w zH_a;CVetWIbp`{qGYa2gbTrd#9`)5{YG|9~Fe7p~jTjC?_U=?T@+}~p{(-g>x1RLl zQcRGh@EBA&tstO`ic@R!D-sMi?Z`D4R0~bD)fP>fiWwul&I zjj1P|d7CcM?STqC^mrOf*I#w3-H(D%domj1fDAG9)7ZtP?4rE~#9|am)5kp%d z+g=>X8=@2qQ4mKI)Iq!F@`6>|h=l6dJnSbbe=MJd!*IfE00JLp+TYm@)6VBjRuWk@ z$?-ko?g{*br zh6;;dS9gNI;}l%;Q{!SZd!wSrZqFOu{FQd4Aynr)QT1ok4gVJ(_y>>tLy%ud6~URjHQt!mzK7^jw-vbB!jGBbfcYa8{Vnr7foMW*4mkoc>%2;~8+Z zC9ao;25Ok;>H$D3t(%w z+Abi?2GZ~3%AswU9GW%Oz zrG?fTuUf?COQNyqnO8M5*aNY#wEbNGpef08c}c(3;cr(0?l3lIQKq}0`N`OsKJnJh zrO%uz$n7N~fW@&zF~HbD?Q7@>7Igdc#ag>QqKpsPGw@<?63TOa}%7VUBQfRfc`5 zt}7b{au<{I-cJB1?zk!T74R*eyYlVlv%ekV6Z<8|$mKG>a|)P)(g28_mb`sCBPvV0 zOlv890xSEz^JwchEjAz1)d^WkWKR$+?SSAJ0dg!qN(qOZMs|HnN2d~11Y-Cq2eVlU zdO=I(uBWtaGJ6S|>Qtw+zV{IFQ=z*gcZ*HgnNTyr;EbL_Luy6jb8bO%s_L_qqHP)L zqChDjBoSBD)((b?Ix=!Pv%DE70VZMDnbWloe58JC0aY?(6h@5YSR#eI3~>vYm&4NvV~B>Yk@>Xjs>aAN(Z*Q`sPR zEahokciaTC0)nO(t(+uR zxy!Bg1-2TC;X2d2=Q8KR@NjKT_g z)zyd*zV1xUizpzQpMhR2q)lCAY-MF-ZEbDA6F+RH8pKD=^jx$&m)?kC)Yy<4*>;Vl z>NPnMofQwgeS1$HX+6YKzSP5t|yikBg5?r$6TRX2-Zo=}a8TFkh9mg@WM zbvkd}X#znxWZwq^hv1htkj;9aw)Ws7U&R?l7CBQoNJAZ@%v;f!ZE<8Epsx)? zGVI=Ay?O7N7Oy?N9~!vr4GwFQI||piWi(m?vQ9uHhr(G7W2y=;#?lxuxN?zm*>1Ec z)~s<6ID^X-|C+3vV*u{Yw*SSGRq~u)PjYta7(zS&`h*6uVEHzO{%++2x7gE5_Yp-LB^#k2Db$ z=d)TJmwgx?-|j#w%r7jglYD$hp=56ExW3mw(pJ-q0gJdCpS;{}G=J)^aix}jgwae& z?-KG5CVFgh>&s|WQYOU}t>NW_w*MTE_MuKZD{!e!JEB=vC7$*BHf;ht&Q8ta8(3YN z>%x^o)|0EJmM>TqACz1%G}toaQb%aX>Ur=J@9Qui-u-dh3p13r?Wm3q7e1b`EWjlbgR(n!fHQ6NF0srq=nb?*x3b ztKe5pv#;r>Wb!h`125!=r7LI;A}oSX{9#=Y|B?gg_`B0s0@#-LZ{CA|sfZP##f zOTB58zWfKq(({CqEMGaDoSY8+`S*KL-jk z7+?_|8hXC>lJt+(`F5@84r5&8=s~|Fm@yMlB**DMfBSL|{UHK2d`|rqM2JQr_%U@) zV}k+iqfl}oDsbocJ0=KGf0mBhwzmF`I~rZ(4fV?l|8`SQ(fB*jB!DpvNilk#f>{M)`lRs~eI`J(d3HVsb0=3#cPjaVB z)C-cF*Xwa~h9Cb;hGEgO3g7%tKDzX z*)c~y7HFgX#CSL`^(VI8ZFY}}jFdAe$9M+}DKj@G-~laq+lPORBP4_pZ34 zq!Z(P^Yxk!dDzP>yP8M7;K7#E5+QMLw@93nq{Gp&JmNcf&g=#S?FVml)^CIW&VWSH zVrcoGLgFBl1;B<7J)^{%$eivA#lJz<*B~?`@a@egb3;Y4l-A8nHn@#7gS;7iM#rJW zN&y0Mp^5SVl3D7;mN%sjK(#Bl1c@<#=%>(~idUW$sHLthZdNPTCZ8Pyi9QX!6`HX` z0Z7+sk7vPd4GjdNl8|sf%@xJxt8UFZf^?h6pmNNfJ=^3_!}Z6HrOJ<{GB*+3_Wt*9Nk+%XNEQ-f`3HLq_O@ zI=DNN>LtTW5jvvaYaRjA4@%}vjsz$=dDS+t85x)4rID>#9t@G57Y))l3b^U0n{nHP zBYQr9l9Ejb+N}t>)z!=BSL^Hs%}x7qPzIWy?a?ky#Yv{R#NubBFI&SNbFE1<|I(7~T=p#0qBf6Mz@iTWZx7Rr_p1!C#ecd2WG=22Y?7<$P^g zMF6ibH-}qN7WsrtfQ9_}2L)^W^k{|Pz+WjyH$*X3z44SC}T&NX)$a#VI{WQtrCkTd@ z%bj6{*WS`?DMs@0&PRJ|1wJY9TMM03yxO)q+!4_(FBZ*_^uTYeACvN(C*v&WBhY|g zL(U#EG$hJN1PI=UXKs4`#};<`QgcZ3VcK9mDI#^HFLDhx4hiC(_^#fpZ3EH(TG7m` zzH5xxP3Tr%yp#>$iftyi4_abCW7bl&y|g#dQ840028~th|j~1tNkdsOTu} zjCZ|T0CHL99nqE@mfLIo{%y8z{QG>LEr_{lT?g{4tE;R1UPBMnI z4lJWVoqdz3RL5n@rXAJ)`U~&vd9c{FQ@N`QqmEh-2+l3hK0s-WAb)JJvqZb)+x-Y< z%ALf&-oK)%(zkD~dwVm_pa0=Rfe=bG){h<$=-Pu4A0XZjys_XZ%jJri&>KeW24ls5 z*F3BTJOgTu(4OQT{E%UvVe|nKG{96WXGZ}xQkbLfz!P5peTUuz4KABIh*qMjUY%s- z@@Zj}IKEGCou=)>>13fqEz{|5dQjAT^dSbTmg$_i9~-;UN~t8L!M+=~&MPqEUm;Jo zLGLP+p0VZff+bg~(1XKG0i@G4$qK7T;K4asHgU6^iGJJ|iE^RBa-Pcf*%HQcPYGIm zYt36(S-;Ewr8B?kD3yJPbjEVAuDV$dy>vw^&*U9v>(9l`a&buBe|p;HJ!%`|rnM=@ zPM4+ScTkAvS7m<4?ai{>!J%LHIb+oQ^uA&iK_%ux^5vmiCj1Ufw;X~=9sO%mFJ60e zl2qgGe`c*dId!O7+_&gaPoU(rlltkr5`9^6~^CIRqD4&wH9K$*10~}YkAKbJeOj?pIxy$-$&46 z09WqNVH&6}UV1l8zi6?fN-6VKskHi<6HlNp5@y>JmzR$A5yCFdf++j8w z`C^`~idxo(EF|y1+;ep)k_}EcHPG+5+a(mhAmn5b*Z~_ z@#UWH#wFpe0niMM!<>&}GSxgZHrTSq_t`N;R+ZH1aqw1hvZnbOqj#<7>z40pGgvOe zcD(*rOqvBgrVyf=VP{XNN8y6o}Gt6$IU5HtCIf1F_ z2LcOga;Z_i>}x|>VTDAM&9y$tnKxQAB9fZVMIV$c?W!<*%i5JLbtC=>*+fxJ{+z_5 zwdOMx^D5dNXw$1cAR#$t?{4z#qS_RFf3u7JIzXdiX0YsCpPpU<8I5+PfCBVX1BahG2Cwh{RtFLCX%s+T z+|z1&m?P3?YyL61Q*7?SDjaD6#N>AipFGWu5LsbIrzI?}*Y-dcpe8j2PemlcB{3QR z{Jl+yco_fP8g{2t$qw{?D6xb}HJCRfdgX!^73zMk^H8e?1Wwpj9hAdDmioNaySHg= zYvHlzN4fnp^dKJCRTv*bJfha&9yNX{IYX*S1d^PcfEL^68dev zN(U(VOZfQiV!(~NOtEK2n z@JSu)`vyHl{aiWq;|LdSYSWzk+b?l)f}CrHsbGl1A0QCtw5PR(koqP3GXN`4% zzaRTeUi)n%_<}rCi|_@+k$J4 z{Ks^L`n)Fhvzzt63&j0MFpSZRtHn{mO|pQX8{(NzcX`XfV`b54)ef}i!4Q;VOw+LG z`+8TZ0{s`XQIII{vp&EE5o7s=^fkBVSf1qGPQ*A7qOB{<8fjfQr}`)qYf_r{&U{B@ z{m%P)oKD!Qm;RC+(2QT>C>Z{{_s~tPTwT!ri#0JX*37I82J#VXH=1$89CGmbmGd-< z?7{L^X@xe%ahM^%9sqE9dv(bsw94)kLszdAGdeu%B2L4=9{*@wHoaeO(xoZs2zOQB z41ex$vX*CVtv<+}EknXWLJv*e((h7V?5D^ZD7perVLMo#P%}gmRS*PzY!B4{U^+F7 zzx7t3Jp)`vTlcnrlNjk3WpRJ+!%(MU54GKtr?p>OKS~Rb87Ta1hIveX@D%+WK|`q$ zFOSHS0yIQ`jqfTh>g|El-iL3!q;u?BAzT_^T9qy-DVev(@&m@KL`e;`3=PI$Ngq9W zgvOr@+V)R8+RvkGUI~ z7>n}|iVLPUPhP!*BerFc1zcWaAv8wvqkwAa@KROm1vKoA)Zr49YfY7D`*6W*6H?DN zY9L))SGp~YD;Nd?d%RK#@lP;2l-rO8@4J`Lcw|v>SLeN$Z;J_=H=%*pL6d9%`Geq<}Y>EI^EzM`KIq33him+T31~}2{>0_k;(xo8mT{g@ujjDUCxC9MUwDQ1ndfG z74bP50kE50o;(Vntyh;KoCuLpCBQa>xN~KDmD07Fji@VmbpsJ1!j~cOQW5}^T6%l3 zqf`?rA*uoGkBOi`Z7HFsa^afQCtrOC`i56vX5e-W$2~F5^93Uwx_OC3uCgDbz@MvZ zU(J?Wat2hkw|Sa0CNgQ8rv23h@P0cYqmDaMLT_{Ejjoo~J*oTd-aAvhgpahWud?ZI zv-9fOCSOV$k!UDpT%k6Qe;C*d6}~TybMQAR_O`d7#sQAG9!_@HN3_nteQr5WdGhK( zc!$NsYd`y+#2D^5rKg7jfPLwmft47xKtVVenW(lKhNr~Cr|agnxX?mfLa`Y-h+RHa z(3(b^_5JZT;nM~Y#<(7mrKZHL&1JBWP?}w+=4JJ;jws3eG!;qHaZzvF^XpPW7PE_# za@D?z{?I*~wxPK<60u__rao;wqV$UPW|UHiX3+t}2k0p=iXVMd4Z%T`02Kp&vs||JjVeJe&?vGFo#HX|l=@qQ zzc3^biB~m<$VvUZ^-h4G^e7d6sd5P9(qt+i6UZ)Ma{oSDBU_q19VkL$ae5 zDIxPy)vrEKTf)jhnt{wOj=XA*#J32jB;6r(fWf}M)WEh0<`!2#z5FGx^BsMcIO6>< z26M!fa~_r{-bEULDD1-g z1;r;zl(#}fLWPqn^wZZ@w;kteAX&44fjniudv>Xa`b&VZLRS_P75QAWiQwce>+nBW zC|T>YbS_K|Nc92P*?RUe-+DAwZ1Z%oc%By1Gwx4*?TRIhi5 zKa*?{INv|wcRyuHixanG`^vu>b0@~jf4yUPL^5k;e4Ob7M({}}jC_u!8SAt$kWf68<(9c>dmxB3<$i6cHV ziUNPjahm4|+3_>A*4KLF8*5n(+coAfabOk8Hodk@w==J~s&Xfx{Y5>d>SNNn;TOL! zLrGZC`2)I1)U-om^3O1LIN+YoF8ydN&k4{Bs3#flm0w)guA{>bF>=;oru@uY_P1?f zcjcTtCs4~D2^rZp`}3kqUIBd^ zHA+decK=}KGI7qUmB{U0yFM zQ?4z&!UvXv*dzh^c&9I~06M3?VLJKi$V+;>PhK=^Ebv5=!)rgI{>$;%A4K?qR|Fq< zgEdh7$<6zrxO>AdF!a;ZV4@7)!S2C?&@-?0(%c^#G4_|#Dcd4qUrJ)+T_l}xp}*v( z8!}{H&Z$)_TIL9xACPZ5{@C7;1V`}fn@LmVlNx*UUsRG0v@uO`<>$)j?Ni+czPd?fP2@DYo+uEpCpYCKK<@z4&4_dCa!cHp(PsK|d4{q)x z&;;*JjwQeDw=-2%MxX`v@#P|A`D+XXd|+5A`16l#G~1*%<<;h?udw^FKZ`h?+$V@R zaqJtIU4szEJQGC;uE6;tk^pQ<=0IZ{rg>Naq=X6}0`90jQQ3h^M}$SbdfnSJD3s2- zB&09*4a4mMgmu#i4YX(=w-LER5iJ$-k_O(r0sj}lw}*(2(7rV&6H-pVm7NAUjd#Zl zE*kW!b&JuZ-%vr&gi(F1U-r!>1qkPKO1pw2PhCW+elUCAj@9RK+@mxpuQJc`vEBut zWeu^_X9ZbEs0>CW5OX4nxZJ8JpCpoXWx63LacyU!7?Jh&ZIZx*tJ8x}FfUwqeDhZq zPDcsP=UXdOUQoc5^S)R@>N^X#kdS8VhV+FPD{do+hXclMdkDf*!oP}PC6^f>Hx3&B zkP!*W4t9WUx{QOA;HQY-Fh|qUniB#+JyDN6JVcP;$D77t}e%b8o z`XiVF{dg-8??^#BkZTCRRDKkvSvEs)CG_fgye{9}uKsnhdS=q}p9grX8Jn>yr}i!u zc28hXYl30v?4GE+Xv7}_^>=}yk0h-~7YvHukfQ|ojt9@}n~gD*yEqxSkK5GqeVp4X zF{D}~oqTZj>CB}8#F!eX9ox#6#za5AjkJDXpvB&23xeAMP0Fq#u_L$V*B^(v_NqOB zMcvz`;-}HIQ!~!FNY5GFx2YzHy3i|z$6%<~w{Doxjyf#|jZJb3{o?t|%nkMRr@p;y zMi}W`SX^8Tt!}Nk>zZPp0x;fC3a+U*v<{$ZfQ~l$CHCn29A06^FFJZQO)%b&T0M!H zq&PMFVA}({Flt*W1L{d&+?+=CiX&=)$;ZexWxm~ea46}1UzG5M233nEIZ9tzr3YCl zLHrCpu1NYwrq{w^F!o5v$(cZ>SaIsE9Hrpt(KgRDDWUtv9Qz-hWX^^G@k9Vvi=40F_&z9j z>R0F|QMAyL6XndYcIyPkNClDj&Y7`1E|Y!~Zt45k5n-gE-XfF(g8DWnqvV3Tcu|gv ze_*+{=H&04tu(q{d(z4`M?V>Bvd;=BPa`T%1-G>*M_~ybHz$!_H>0?t=(naBrg*n5 zx`gVp196N2%0*plfIwk`+6Az+N8c03k>jvuKNy57ir=2m@>2N3^s(n=%FhL~`O?eq z>XJIY-T4wpq_>f_Z+1$s4bk2KZ->y7i8Pkp)fj|Y*Xc6{9deZN1qae4Qu3JzD>QOs zMz>d9t8MAE|HIyUhhyExZ{uxkEostLB!y_;N-9KTXQv@#lvT*3(o|BEJsKo4WfdWW ztdv!DsAR8j+0XeQxh|vb=Xt)r-v1&<2BCL`8r=FWtJV^od9hW z0suoBnzM%oze$a($=iygTbQgMzG?4=UA2LLpMMIMfb9v#)W+ne;~8ivTLq~S6OdE$ z&{2oVKeA<+s9jj7E5NMyL@BNbafOi#Pl@2Z`c?^}p&Yn46KM0WH`5zO^aFb;56>pwp)gL{%a z^>WIqoJ>*wsJ|_k{gED(^W|581wiuT<)S61fz`KsPiHp&x!O-BcOpv01gp)*2zC$s zJK1(ofP-w;n#4a@{upMsC9{bT<3ppef~v`bUCcZ8Fuj5;weeMabnTw~QP>V(CGzSA z+gf0pe#+jf5Zu0}d2Q0oH;#O!$o7R-n#d+S=|hc1Dxd4Vlj8?t(!Q5WM=kvc$zSNf zp&StH^C;_JaHvWvp#_Kj_ZWb=a$~(2O&~XXuGhVw5FUJ>{-e#I_U1@OzyU&;6p&D8 zu$M3?&EkZP5aeK=d0i6OwpP;TgYaAm=7O`iSHzOPLlcDqEk}?D1kC007@4`K-&cML zTxxn|rd)mCuOvyL^z*B_^@p1Q89)UCeWx7+94fiQEe-Qr%rCb2E#uLpKnq!{@`;^5 zwlL4>eDOKLtZx^yD?R&gV#XO?<>2ECa$S}NruWzeqvKd|iM#&|0%N8z4A>^30Xj0K z*Iz2e+{Evez~<+7q^;gp>iP0q;oH~y?m%B2Y4yFyWc2gT^nLZbtgf&MN_t%6`a@~g z&>yB2F2nrH;#&v`!08*krIo_8q_FToz*q9Qa_PJUUVri{%J;Qr3jwp^Ws6Rx#ff|Y zLw$cP#Yx50jm55YHVSV}G*)^Gi+~YqKV}}i) z1yhsNgW!#Mh2L__;BKwTQm6;wQ)3rT=+t=Fg}ZMQE8~Ej*@nUfQ5r9EbVw#@7rio{ zQC9*0EZzT%qbN+fHCv3s65+BR!#&Pp@1|e4mLEEBSOZY^#r``Q0aW!CXu%AR!^a`i zgE@wt{<|F3zxB|6eMe4~$|y#j-OcpJ@&2(kXP;9Pu`LAJ=>8LzIV}EO)F)K6aq9fq z(L!tnzz2%RU29`{tlR#XdVO?s-?Bn=aGQOlM`klSD&ykH$#Zl^L)ik zX}hJpuI>y&i}^rS3%I2O2r9WpyWRt0!pwPIXa%5WMD4YqPcz@md0!>yt#A9Fb)wS5AHFIEQ+5aeCS{Ti2P&Ns4szQXMOAOq2l^pYwEE2 z&*}TJm_+Afw)g&pi4DfX$<~+;ep9dcxWFpbIB;p2UQ6wr`YK1U=TB~Gduu3rUAAY) zXA#vBq(3wtEWR^X2%@>9s8x-4PW?(#zM*`1+PtG;izW^gfRNYFJnC3?ihvyvCDDWp zG|~F_PdF4S{pB}@a+dV7CEa=IJpUC;y3 z*3|gdM%qjFU@RLUk%^P159kg0SzNN67v5K-sM4&)*c)fn40t#jR{|9qv4R{h{^1NY zgIhZ9CfP+(ow08`ea+uT@1vORE#}@QOq(qF0#v3v$hy9nTns~UY8d<&>sI2l7@cup z*_X=rUp^|Mq+||n1L{Pd^YJ^{1uuR4MO}IHz^LMI%VJ=SWv4mCGy(V@JAhw&Vuxl| zj8#5^nyTIIh8=C6=W;6bH|fRGo|*Ah>B&##DWbhO1T>LHLKsW^mP0x8(9sc-wqZWh zy9$Yx{L-#;x_^DT!iL!gy;K|Y<1bIuJdAwe?_a+6mRAO{P3nO5KiqEp${_%rKp7@R z?3+JCF&OcC)Ogjf%SUa0)KU$jQndFP2nhA+{Lg=3^IFPqW;SDugjdhm~`m{MRdyrj>_CWk4lp*UyR(hl$DGKH5^b)%rT}7aY=T&cr_d+ z2^<0dwc#1cT2A@AL4N4ezk)pCDZ#6$BdW`@ukS4;MjqvX)XHm0+RZTd7AYf&5V zCdZ9JNp_JQGvaN$W=<7cg*x{ADnhqQK1k`LCc`DO*|jI;x_29p^ndxr zJ8D1)O}pfyy5{9BYdYL;t`vtg;goyXJ|Np5+DKutr~?6M z13xCh#i1Hoi&88~G0+SJ6JRgt1omjoGHz7QTp^*wbl;q{QX+<5*%kq#wf0L#&L zMt%*goi||fW$y&Pi%5B%f_5R;C6afT<%gckny(g1edWs~N#y>7P3xBtIoNC4qUJxw zc0%cbsT;|J#tC9)0&=kGE-L%>f?%lj>Tr1!&MiT`?0f2S^uZc!3YEZxZmR$q<1xqAAJiM6NaEZMqencn77ANtBN0ZqQlaaQH`7t9$%x~ZX1oDSqI7~y`%>|x#K?=MK5 zsgkn3h2p*uiQfZSHia^f5S%_}2m2w{X$?k-nL9c;900WVO9;njy>QKS3oqfwyoCg? z#IIsfW0ryXl_xS9H81;JKZssLP=;37CeGb0e#a@g4H4vyAI`^raXq_sc2fRl`zVoc zhMfm?f%3s#-%(~|g?5#$_w=2o4MeXso^)5bO6sNb&hovlMIB$gRen4nDGMC<@2hbo z+Ck)C48J7bTV1%iMY;374+o*F6coD%O}wcsi%2+7TbUjgxldo26DCfipwoJzEfBMxYpoO{ znEjU3ynCZ}a0+2QSbM5d8r66B>`SBqAqLN;E4TI` z>vD%e1|AWNzy2NdVL3nv@}f*mx6#o%Zc#Ods*WG&{8l6h*aMPky+-{7mT zNj49nI}(J)+a%kE|LrYM2zQw_XBkLNeHDyxhnH*RHR?&a@f_l*cv)T6AHq6arSpD) zjn_v}pL0k{#iq7^fRG-r0;|THKr&I47fAmRx?)M~>COao3$X!tFB4r$DK5ogbP9g0)H1=1fI1SaG zv?rVwThv76_S+Wh_>Se7|MFP0n8{c6`4U`o z1Q_^As^|R*7gfQP_Xi64nhHCYIu*$2qrL;Uk8@WJ{|v{5^>gQ)m#P(2y2R2vwJc~y zxPP|K3xDbZG&e2E;{Bgm7xg<`vu=qv97nbc4N!?n+b7YY5ZCqKlS_kWh%v|6i*Ii! zNoLRFyU9T;pxlEkwsJmY9h!`E8AW=pdeqzLYEAB^LHD(Zr>Vq@{ zXc3=(il5uMxK3)>`B!c0g5Ek%E)%PG)ksN=I=3wnR6IwoNDwIrWEt_rj?5KAibiBN zVPRp2gg{0j>;QoA$GcA6A=vmUP+Y)WCuDy-P{7hb#x5Ztp`r1pxUTCxhA$G;gwnOp zj4DPi7iw~42qVnb#=^T}+SO+`Ua*RB+5OL7?%yCR;l88U+d+a=%%};(%K&)kP|ZVT zEokRbHqi$RimvLrp*k253%#to){TCW0lQ-giSAM>}11WXHL z(2&#L2t8ngz66$nO*pMjjr2h0Sp-?hAg&#R=6;OT;j&<3Dh53XMX&}8?S)SB=QvqS ze&)7kD83L(_pb8%wL!Z8ll3MO#vIqhqnU7wfBX-2%Z@vo65Y^8E5BPpR_8&blOf%Ufy$HF8PPT(wA}y zed6#h?3w@_*9SrXgK-Qse>utVoyVR6tUZSk8R6GU4EUKv>1Hqah>#O{8T&}Z=H4c^ z{Z1X3Ciya(<%OEpezY!vrIT@fLFI*Y54fxd2I8D$-}nPJe;27;B9j#|+w+8v zl^OvhjfmZcZd20L4-c1 zWVeFz#Ct-R4KdZ+6*~?aydwlK2H;0f1QGHRSK5Y!Vo=H+-S!n1gy3PK^(3aNd!j!f zFfM&N^&WVW1okfH>kT&OVZFwjr;dv;C8ZOEM+0NlQWihu$Og=^1{{I#%Bhy+g{R%Y z{<8jVE6j6#sls*!`%=o7_f}NCYh8Y=(Z!|`8yU9N8URk5(UIQ3LeI_ z7oPUdQOXu~;q(E+6`gUM2UL2V#kFtX*ad|$q9*B)m>;AJ9+ex!*5%^U9~0o*Yh-;FtwYm}6G0yHy=6Ob^0fCiWS29Nu;ahbJ!>nG)V&Tf zh3=VZ8N4l$@;LE5ljAnKtG z9!5qz!*0JRmi8+xp(M)z9*j*dMaS@Ea%%M<;PNK z$=+*2`IVMUdaZH^truQk)1uMar{jcJ!XJs0wJf{=nN@eJ$=JiCK5o$*{IK1zF$fjU$U`7R(w_m_D}fxwjgL7_rl zcf&N$0^m{_h*Fy0a!c*o5DEmh-*K6EMB9LGg>@6mOK$mmkZFD7+wV^5!Mod@81F7g zl*wYx892V_N&b(=s5uT_9C%hOf?nb$RsZ<$V?tviquztS2uMwe5xOp91~X3}5R&15 zK3Y*cOW#RIU;ACX!kud;_{IZGEEuuH5kQ&8VQ>ogNuHfB&ck|^(Ef*(;K2L>S98hW zqlQaWSNmAB5G3=!9=hG;FW=CAt~aKadCcTNBYA1ME31M-YNVfr)y(vsy7cSv8=&;Itj=x?bNqnLG3cQ2XAF-Fm+^2q0OVH zOc;O{60{8og9-1JRjAOzjn+2M@1LK47$f7U*!%4i=rWMSQSj(8+Xv(jOeUF~*8yJm zEMAb$ZwWk}SFdoLE&TSos81cHXtzGn>wW^_a9_Hy-^I>g$y6YuZNOnI<;yUzVw6T?Y=d}o_fzJsWqthsr$r^<&Dh?!ah z2&K>fiN-x3Oj*3rSyF#HVXV_a3wxDw8xaHtu?ib6jEp6sz-P}yKJcAZ|FkCjFinK6)IQGAuhD^1dvT?cJBGv&KMRtWqu zO20Ijn7eF|<(xRU&kJP#lhaterx~he6qq3l+3d!^w7RmmxTVax@e+1PlBhS$oKMZ^ z7&aHEs{u_y#Gwxdjkp!Giwr`n<2HT7QQwpbpaAq&r&ALXhuu27%tRYzn;FD4`hzKt zj%i-7ljm+}y|Qrzjr$!PlfH|fYy-QqtheDDrFdfm&8KNOLc?n}P~aSa2fVh8(;qGY z|Le139<+g~B*69m7Hyy~36z93UZ5&X4*MRWSGa=Z_5bmkjYc{aC=o@!v|Lb&-p=+1 zV)HMg0zowDds5K*~{Js5g6hcc36aeRztK$IcdyDfj$jZgWS;ooF}7<;!-d zPOB1HOaewrh28*H8{2q~eF6=aj~M_uq1=6ok%I{xv*<2s&s*D{hsM8*UXRJP2D514 zB3h!^@SP$acdeIsjUN;fw(EX5ybF4}p4DB~|Lfml!8Uf$2}(~zp=7gB8-3Yg`&lD` zW~$29aO~>jy@*tZfQi41KxJNUo&+N3KPT4>HK_-WgUhYCH|=ZPGBt4l$8J zK1qID@RMMmDz)@RAQAcJ$7;H5+w2%egk!{8bA zr+4=zccok$aSMMph7zRb;ZIWVZIv`=>lpVHOw&a{xsAN}q-8YEM#J?W4g@>OLW(^8 zWji=kebu`}rI*u{pcJEN`k+ZAkGwon%` zQM0s7zC9R;Q`8?~$gUKIL4-lvVE3(~zsDaFrVyk=80MG=*OVyX$k;0#B8`a{St}3p zm*_PBd5g|tIj=ruxkcu(w=^Q#^0RZ_s*yloBH?y59h|#-n$YGEa=?$@k4}r|0#;dL zkcCX!cNwucm=Ddn0P+k{w!nu(nveF%@n$Z$)4~1512YWnW;mHR2Kl)^a6f=~<%pu+ zew_Pmbsw`ajccd&qv;nGlK@U4DZJhxytHdcwHby;oNVQ{>@t?iK0n;o+u!8n;9!KD z_@MF!J&1!?ur&nPUUO<&pI!fz0I&=|7v{;qV8p2&hLfNiQjBaF-Mfa63#?iD{c8Sm z_2-5DIS;$PUqFX_Ypr^}Idl~cc>Ab{Hae-{G8AHHO@Xq6e{62yQ`F9fx_@7y*OgOP z+Htemn^zT=`ZZ!yN+Y;Iy<>g7Dqk_;DQ<#;-~Cw@z@{%hS~-0&NwJ#x%2QP)&*W?p zb)7Ilr%805pe(Vk&3(G>wA?M*30V{}TKv3sN%N}#*{?x($_>!e4F3-qrd%OF6V^?e z3Qk+_)G`%zEht?ZL$8hpE^T41dq0)V11Da_+TQ*N8eEyHd(H0YEP2U?4oaVvPy5Ac zgl>E0PUiNTjjFnKyQ4#t7MP=@o)BwQQd6?(#4JKE&l`#Keb)L1sYqk0cD9q=aOmls zDoDPQLvRNQ!$g-!Px;ro5$_|aSDTEwL)MPU+knl|O)vEIbasjCpJ|y!krip)}O zr}Uy^k9^r+x2S7|X7L~APPye3_=`GnrS|#kvz}9=u-@B2v+gxw;P6vMD0dFfYy*40 zciJJ|5^r)}=;wCbqx5=jt_ZZC3fv^n1ExZOa^KlSs*{fp5=U|+xOa|d2(|RqS)#{b z(2oo7fz*HDblABv;RkQ{t372@j54>EqM(a5njDGUhr$ieL?74J_I%d@V(ut>nSY_Q z92uG5F5*V?+3w(|{dil<$$XDaVv&Bf_suSbc#K>bYd=O6Ps_tLzp3!WVgc(<0H;}| zqsykQde+RD(3R8I#zb%)XrAe;X7CYkGojbBoT2^R>E&lM4ZrFrUn_hEHI9JW`ha{N z8i6v`#YM`euYccK+keT<>FhP>+*EDv0|Y91uSVp>9f?k~2c!SP3ytMFeb9q?@E#BN z^bCIVd7G8Oeg+t3FX7g@+b8oG(B`jbJZ6oSmbok?J~?#0rb;BT6!#R*CuJcomDo{!1-WXtPrk4Y zsJ2elsD7=~Cerlme5?G;zmM88$9b<4kCdv}t&6rmxKlxg+7=>I*~D@)!sNL4$vDPj>s{X|wMxlTK!v(`LPS zzeFERfAC{;SLq|g_q^VB{oIb~Egk|p!gE}~L?q<32E|dF3Pkx^-ma`@)TRkyUf0i4 zk6M#VbcGY7G$O;~M11LFQl(e#Jx_F^X0O+*m9K<)_y?emKfy@k$_g=scSKJ=(GWsF zs7h$M^+#8J6EO)B8w8xD4C;tn?UnPaBu7MA*D2Wy+X&qnqU(gn5*<+mW`$NrGGwb! zaLR;vv;^Z@vo2+IpXbO#V`PYv@u}ka0wlq&sMEla5!J%Qq_5*uu8H7H;z-ZM@e0j0 z-JF$H{T)`BTj<@p4B;T|kd`#f8|a4w5>~|L_U&l%WNI%@ZIlOV+56LO)6-AmnxFnezTfo3tTeO#`dHnDE;RbSHCA%zvmyr4l_@ zpVVuTF}=e{9bz=vz+_KoXlMX*GiUYGjvk{~x4$5(%gooHtg(%!RaajF) zbltm8if;DGT?D+W4g&;^G108J3sb%>piwFP9;QMw!^0529F>{IGJlL$rZzc)G?!C0 zUIgC1OuAIW0;QYei7>?rLy1nvhR+dp4|EN=?zgwR#E{ELVmK(ukM8wa*%Zp%d z)>fRtR1-sIomvT-<-|o{?Qx+gLyXMO1}{6RC8^y18`a7GdW8uaKgQQmGsS6O-_8Ft zap(u|vQFn)eQ*+u85k2C&VW0DF6>GHUK{p*e0rdD^#8=Xj(2Vdb;FUUDOwyJJ)&ma z4q11iS7ih`_Fiwi8uVDE_J0Ev9vhpyvX}&+QqJwLw_B}QkB`J&|59ZGD|0F52@w=> zfrkvVll&96{oPT58AQ;e(nXAne8bD^$I{A+A){Q*=#6dWAm`edE1~-mER5rKc0L8J z0UA+R|2~aT&pq14{6>5sha=sNse1c%aj6omEG((xX4r!_1Vx0MuQC=sUiS~k;kVU@ zNXi0-gP7S4i_;h&+*ga%lGNg|ogXQ}s~kpvu4AKz_{C4c=(G)jco~EN?tdWqW}^Da z|Eu?;Fp_Ng1nt4*&PR1cB*3 zQWFgJc>0*a_sv%}86Tz$VVkncK#12Be9!rb1Ke!4$OHk0X!uwW^WW&^UV|d@Yoc>w z)iY2@i%qx6?%D%fwYn(sxtZ-osc)X!4Jr*O5|Mv$gpn;dvq-JSzn-G)tcj!!eE&3E zj_xC&h5M;H=qM5Y1AXPWjvKp3nznLXP7gEbrX-O}^uBsUnKJYVefh&YrLFUJwAv{y zT!~ESWBeHO8@gIM#p4h9)@!v+6l|7MT~6^0E^%z2*BIHzGANT;eu_N1=h9Y4T+*}> z{?Um%(nAOuQ8x$^Rt|ygXziR0ZFdP?FXGRWfSUo3Vn9^{WNIrf0qnY;4DFFla05gtA+-A|_Ax_$ zu1OEU_Wq+Y3_+^Q=!$vs=l3q(NPhFC*H2F_v8S&C^8w@Oe9k7$;eFNzY=3|3{TLDG z?sF3z8kLt=+|OJN!_d~3XUg|#6IEJ&=EF&trg_gvu}h@+_?A`s_8Cu?k;9&3l%lO4 z^yP1VaYLwitpsKkuKq58^qDIZ`XPTg`yl9&GJ;>(h&FSfw1b+(OUOq-13*)= zlAXN3IGlddU+#_4x#QA3l<#AGkVa!|y;jM|W}&5*$CXec45VBSK>>fd@7_Yq=7mNy zzdG{EynwB@(LY0eY<2W>s+2XR|i}SHWxZ4qp)^$&yFLDkIpC0 zTh9MFY9=LUY440aV)}4#zOD;n*~h>G(fdCsOjK>V*6=j=z+TGPdE`GIp3vkLJM*VM zuQ9L5_5}%c)=iUdHCcc8nVKgz;bPe@xjhOokD~no?#&~wxWato3iG=IlEJ}~CiFag zJ}IhZPRSuv+bd0#t9st9=~=~NZfL?V;VZ8t!$&!TpZYaA>#i`sQ|3XwhR~w;X7m=j6oUFL>3k z;pV3;5?!nh14=)Z4Oa+s6)@H9>aRr$YocU1qyK!ywv-;fdj-=Y zpH#p5#&aTI(s)aEGGn<}x_N2zA&Z+$$Hk6$i6_u2Vl+EcL+9yw!IGn}?9Tq^A78Tn z!ve*V{Ea0~V>qX}H5tTD6+ez)mKZhxZetdxcc`aMv&hfOvxFWAWS@J{U(*L`aE*C+ z-5JnfQP^Vt5(|(j(=MLp*y;_DtCbm=pmjLvEXBW^z-C#hWfNExQxvAKv82a=edltb zKn<7&Qm*`nk1%6rnk_Q1-tQb%C;&+0Xtmh2-n|wR*Sc7j7@XbnL;20$SA;=G4$gWE^w6)*B6Rlo$%Ix z=4RmtQ9srx^umK{kB*xvKw}st%fv0=yj3IBe*IHo=v30zcx&rqdwY9q6vvmo;wpj2 zV@Wngi*#35)V=t#Z-0AUC+LA~RW{@yq;yg3hvNQO)Z9R^n7a&vhy4Qr4po9MUC+4u z_Ok8%=zENBM@-Y5$lSN}3(s=i;xj@ab1>XK;_1`m+d;}AgXd(7-@_G~0x8Klv`5W~ zKt=0Im6sK{Y*)52&Nce2Sj3%=l6b|$2s1~nlx%W%r^U!J!lcQk&m<*7YQ67^7M@Ul zjtf6Jzc51ytq`Q|_ho)$xHz<1uNpALIoS&(`J~j;^s_4w znSDpi{waB$H;aP8#JIC1hZa0hb*C}Jn`eb6sZa!ErgWx=V>ncvLTt-}$~(Ka^a$Sd zR8fUNFyS3##X>f}Eu=4}69C9_qZ60R67*IjSG}5&4G)vPj%_yUuQyflJnEbt+j$fu zf&Lo;K0XM&z=>69I$1&3`ljVpm%T7@5^cQ1h2#Qe^4A*zy9q<*vWbi{g$ z>22JdzvTc(#|WV&Vg9+}eymQsW$D(VZ;1hCRZp0%6R$dfa%i+OSsWf7zJ2?4?S2jB z@jvw2)7dfzFH*CjU(Eo|Gve5ywh~XjT z=q8{mDmRU8Mx!?cP&db>O~o1Ydw8V(y5Yfy!al1*-FLASH$PbyJYm^8oIz|( zfNK~uG(E|TOpET1-j@XWKVP!S;IFSTxEMxK%F4AoN#1g7CJgy`n!+QNxI+C%sAq@% z!V8ax#~S#?JQd^W>?A##n|FAl3hQXzpqx&=XV&Tb)>6I897IMk;oBj+rt7VQtMe>R zsOp*i+DDTpvGrZuUTfCK6VEv44F^|0Rkj3KwzjVakb+1I7Wv{VrLL~7s!9+}pbc{h z2*MoO8S6r_te3kaUuQCZCiVn9s}-;_TQFU1z#5(nNkqLNt}E}kDP}Bz=7%P5!H=l- zCeL4cz&V+Y>HHo(gEKux6X{(xtf9(<5V27h^M|ZOo*Z$}h@EorN671ePH4oVkg?V~ zo(uv^3>u>luw+3Y5dEBj$DhtSR+xGGkh}{yrF-}Zj8YzN5?2c zNs1sxyaSDK56^;!iD?O?v6j&V?;FkB{KOZ|r#TMeJq!jJAo$ljj*144=C8VfoOSa; zUw~lRjz#LVF0m^2Rnj9QXxuA{lbwOnOIXniM*3(=UDnz^DNcIhxBcON)M)*ujIC z1sG!`VE8=j63ZLCJG%Wps)*5m%RmjVf*PxfF9iZWL0(QxmCVD*JM9}07i+zY36=*s z(mMP1C!uOiq#oSRqV2?lm4+`bDltJ6A~q|z?+k|G{TIHeX-}3sZ-ElvDGw8Qj0pX( zqjGjA{-)sN_YAJA%^VzFXmTjws&8FD?s+ndC^T~f4!O>NfC+bbVKj7vj_<233=b%> zDbCH$&(F=BqQ3G`HMOM~1{DVW;)Fkw=@$)RwjLs9kPVS4lyDkGTa(xQ**9fOr!jELUHsvQaRAW|S8`+iS^~v&oPRZygs5Kg86CI*7*H0*JmsY84Jyoq z%7zcyv$NYeeWB#(W1E(qE<>80P3?84tQ#WxK@)jw(Fb_K`5TGo{jVlDoz(E|-Ye*I zlACX-b1w%QMj3LS+7a4@ZWp|NuPBX${cY-R5DesHGLqGzP;0`ps)0TkG^t{E4CPx& z_HP`8#vw*254e$cN$b22m2+Gup9Myl8b&}ujXeZuUDI4@w@Bj~E|d%6%I#N}e+%MK z9X~%k0b8m^%1&s6VIJ}WgI4^LF(@v~kie=T5`eM2+k}>Ji7h+;1&V+aGvy?~F zzJ)3#-cvf-!%-t%W`A|sL-%c@kEHsB-*#oBD~A1{P7J2{5YKyz#%b(dLXBoE9>W2S zwGXtB!;vc*e~>;~4!(_JINDK*@%mX%kcpdgn2t}d@9P!u7S!7Z2+n75ZO@6Js5f}h z-~zt>neB6{=>Fd8UMhFK@{iaR>02h`m!mxE&!Y=JcK_6wHF9T~IZ{U4@)Q2xRaK*t z6z;I-_EIk09e&TBKMxLuHERVVM9fgUmR!8xOI5rr8ew-pO&!fr{Fv8ZdY#DEDNW_j zXWq!hW`SXG%cSqzjS`x5kH#4QlROye$}2VO&4)JD-VcL-zTk8kz%D=%i^oL;f+a{# z3c*2#F=df(U&8(LkU%nIJ1|r8jb*oKs7A+93@^Prbr$}z88qL*GSgYitLx{o3(^|z zrpau*mrEB7zwE`bR}I8AbIPGb6fj0cGFmfqcT^XI`icr@9tU$?PEHQaw@F4`z8$la zxa%yY0I^Yx*cM(G(m3k-3FnZmPyJ58(Ku&L;(0(`fJiypq9f$XwYzZB)8Za$A`sc2 zm9a=lDN4Q0RF8bOSoY>LJ&9osMNZF4iF}SL`+NrK%7PB2QN<) z7Ks)TyGwH<2I7O4Zq)I?;Pn|ijJYzBLC>B!ldQ-%lh(p9E;$+cd@Bl>6 zq1BOe$Ef(+d3k=-M#elpD2sr{<fx#I5_y}Q_MxrXI~{h!uk(8t-(lhjX#@w;48G7AntjE`J<87UfsMo0?8Qm zK!YK6xRQbbYMUK%nD|}=1_q+JYn|5#Z;5@*!x=A)-x~ZY71bBVk`T-*C@K;SKnT~G zTqNxcy*Ahl9UAI29gLWq{d=GJLL3a}0neJwb(?CrUmp?}KZ@Cw2O8>BmlRHRfJfY| z$0Do?G@>6C#oNtsss4krtlhzIX&RE(LCke<@0_xeCCM;HoDl!OJlE_4DJzUJ<_oWg zXCaBE?K-6IU^jF^M(jmG9@<|p37zxC9*k{X*U2LKTGAn9oDWL}v|B$!jD?m@2P^w~ zyXI?*nc5GW0V!3m<)D4?mX~>*koZ1YoMt#LerT7D-jSP3!x?vWfce(L|1ex$5I6#FywRayGCKyXJg@ z8fl|G3%~a$R|f_kG;LleKVcQI)%Py^eHER~m_Wmm2xo!!3{DV8JS8BSX6efJbT};; zDs&R2`IvwE6HnNW`#j6O-1z3;9hmctr&DC?Yn&u&0V%H&OzB}6Xr3>ocd7pxe4~D^ zffJ8}jE+Uf%DTIS4LPPlbF2J|?sFvAB}z=(^Sk;vkcIfzdiCJMo&5Ut5|U&eTc~e1 z1H!tDo`L%&uEiUehpxqN1~jyZVy~jmK@UBSY4d+#tmwM_b(=c0Tbze|&Zrx!Lz>RY z##UPr&D+Vc)NkSZr{kbw1KxQ)ai~?##817@D#-b6ANPrX;mR|0JUVc%_bV)4Mw;&L z=jVs0bJmRqf6r@19zU8xF_cb>3wr&JSV!6b`@fs!|5?PXK{!asPwWhZSvAjMgWsm- z`>1m!SkLcqHYe?=qnF={-LZD%x+vr73ZCsoBQBR%(3{TGQCP*}fbM&9Nl}F>XzaqR zt$~!!G~m(T+OL<%^S*^KDwq!ODc+F zu=wMJUpY(te|cmG%~?jT)c?HOn`r32V`tbtn+NNBA$jR3XIl>S^iE-9`DpFSyTEydzL4^kZ~MDRyx0COS#gy9+<-iWOGM z%F6?rN{qPf(hjWkX}xZt(ACJC*f+4C4ch8%KEt*Spec8N*+PP# zFi`X%5|UZ;GY4SP@+F5c6c>6z2sO1(|MuhU`Vf(aFUOm`K(dOdBzbmeQB@tW9sI4! zFf{0-a)ur96SWY%4x*VHL@d*W;?7&@J;zW&6|6+~unjHRgk)aXg?Zf9d@M!F>%%x2 z{R4st)iX=WUK}+Ub)!Os$D@#@?7$y~?vJ|t>t(jDn0z^|kM!-Xka&+s%%^h^LEvp; z=1Sa~Hy1EOidZI<7ssx4o<+M-#7WTf>x&}kBkDajc*!DULy;Zl6|FE{my@eJE%c!p z0Pu6}8kLVr9NX6qEFbt69uO;GU|>K~DKFhQCj0gW-YeI$(QF`$wpftNn8s#*1?12& zn|J5fM-DO$qJln+usnY-{~=UNXWJK~u;*f1JwQn;V6RiSzS8syRugiH0@~T$H8nbs zD?I+?Ru)@>Nau-12Ny$MBaRrng9HZx00(X_c|q3f_A6u-b_nD_6JNe zhbt{qDAI%h9BOf4pJn^w$JeRKB%{!qs}WXSG0%QJqC(e42LQK%|Nd3K)9g=qdyC?- z)MS4hroo&?!BXKL+Z#g}8Tc0s$D?E6TkK)I-F5r-(_j5toWEaTmR`Srh14gZ@hWhG z7cNRC7ujoWE@0oNt2<6uwmS+@fAI*v42&A)2wGx*jhKMWV02cAp*SbM#Vqf?tfsw^ zCBzWW)GsF`B_%H}j}!6gTh85v0_Dxr?@CQ+BY;QcIRZ61QK%JL?qQOz2rUa%trqR~ z?S)E<{`OZg;mmE^Pd(#^QI!is1m+Z|Dc>4ztB0(PU}1q>F=MWD^?2Jo2q=k%2M6ne zh6!?Q0ti#6q#jj5=8BQq4sTgVAulByioUJ=o$HU@xWOnG?9}J}{uq;W#LR5xh&*L< zgg=m~po1Rjt`0p8w{ve!&l#?QQ^)3mdD@K{8wNJu#fLvyRna;AE0Ska)zj)(_3U3M z>VR_^S$G`23xfrO5mm=g*NL|FIqF7^Tk{ z)EQVa$9-xjawy%?^jWwc{GZ~dd$TzXD0J)|f$iE@J<1pk8V3zsX!gy$3deoDB>v0C zljIW%-s-#AqEPvj%d0a3dNnqVb5MQjn4$_mHGyvz8fUSvso`E_Q<@O=E{b@V!D76jiLHB&ChZA8U% zTm3E3Zj~!u6~4onCe0QNmb2!N4Bj*!5UTXuVw#1d7URcKbORMbH*A-xD-37rIa34$ zU`dAq8OhKbN6AVTk1u6in`{>;W+a*wBVH6L`dUC>+xL9k`s>r zTseCgpmJhhedr5}aE`8`4g1Qk(=CsGa{0c!==rL!BD^B-82ERnIn7#@6YCt7wV$$y z{F_{ipoDhshsX^34YU(_V6qwE)tXHF|m&bwu|_& z-^9>ja8y+x5gGAmMJtZYtcXkvgHz03`rBsaGaMb52 z)HhO55`G6fUv!%CCS6})_#XS5P+oPN)T05mCNa*UtE=nbSsJvt&J!o|C-iZ&-FKPz z>#f+XBMadr>v{mzZr091s6p$Z=R@%Lk;1nNONk-b(BaKPG|u&5N1`sBt+u*&VO1eV zx$huX^2FZAyK7RkTikvl1p@_c>ihqPGSa<%>v>Ip(i;{Nyib5A`s^b!)TFqdr$)!o zQ3okElQ6bI9-6>F`5+f#2FIm6sx}I~K<|hI!1zJaLPiHGaS941RbU;TfxfFR!HuL1 zmzp`chV9&MCva@hH?Kj^4opn~h90ki9QjpWd3u9IhE>6qU+r^viw`f$#{Ft78)O`X z$*NJ&)#tlE+XJw9TQ*846E6N}AoaJB*jQOFf5}1m2Ypm#wE0|USx9xyhL9w}z1zRj zYGmZ;1nKD^08-S-D-Fof!}o(a1_RMu!c$ZLnF0qJ+bpZx%8%5Xbg(&x!291!=oz4| znxOz7L*UWl^#u&F8P`!&P{(D$|BKi74=(>`F7clbe*!s&`c>e5lbFd9CQJxnu7BPn z6E+a!;Xl>T(|~`U$wa0k<}AMDN4L3jF1iEUratw1bl3Xuy7E&PgQlND-b=Y8*VL3) zkbi7j+r^}!)nLAczUAJ&$Z(#`x}HU;>6GQ=bgQp0FPbSsf8@;Ug9aT%f2bdN-& zh3~z&9S2x!FGcXt)=__K^56bgx~WO#73qg{<%<-iXh`ieZe9j1vUO3II-n1tlGpI; zkqcWyj)224i`^&eD~r6}UypGX-kmaEf5t0*t&QJS?eMDng`BmYIO_Kw@meZwDp7(b>2bz3>pR=2g4uW_&AwJXa5R&HzkE*Th~qo;Va2(+ zkGf|I8TaN#&PXiXrzpHcE9wjdicHE%UoG;#d0{%dLzA_vc7kL;%-kJcFQO2uYW!(G ze0i9UWyk&(TO-#tbTd^KLpW>jc{t|v@@RZp;Pu@Ipu0TN(1FMx$ch-s7w_j|N{K@~IU7Nc*FYxfIuDb+&hf}US##&xg7-JUTJ zW0*k_d7wF8n?Xr2Bf0~!#m$5iuhL#A?H6LovYV|0%EQL_?PE?|{;{2z2E@o?IP|S9 zrvN5>hHxDut-jUe+--S#brvI%%hDgr%DvzZf`iV7JQJD>Jw^s-eYirm+J$$e>(q|= zu8yacWj4?`{rTm^6B%NXMqGRN^9N2&k=i+$h#DmMny-|^U-1DHz4H9mOXdBS?)jBa z8qUQPq0b#72$mbXIg7N#v>)Zer=ZmU&G0W-xc8Q7;p1bStuuTrFa0F8!Pxvt<$_q( zeuxuqQz(KX`TnI?t<1bO2%!?)O5=|NM+GA$mO*{J*$TQpn#W=De_XP{{G`l?O>Kat zcI1-oc$Q1+^K6OhG0f3wSWHHef2|{~KB8zY|F4hO0MyPj`6ZkSI-%N4coBqUVC~!n z0=82t%UBt>$C{i>pe8G!l>H+%fr+Jd;ePvNb|kyt{&BWG^yp(sjH1gAAbH#8pVjx7LQC=|MDhY#G)2j^{xZ(h{a*7}OO9CH+l zP6SnsIP8{P)xIB=@6aO#lC52q@PwN}SC3UDp@ ziQKL*In3#Jt3AlZT~bdl6Q;^uY|A!Vum<|Fton>Xw*j!4Ja@-Y96!x&fR(R*KE9QZ zl4ZOp4kHotF$~IR6K^U6a95jJg=R&@wgthV?x=UUvryBNiSK+9qQEGy>4?x45#;Pb zt`bM*xvo7>NcQY(t?H=GVbRL?eshgz=2l$SZ#ZEm)o$lBh*%zd^-14371SgyYqO4$ zbNxo7n@ypfP1dt>qd$BBZm2kg7($V2AC9h+&<)dyG_MoG6(SaqC;RFVgC-(%3yu}h z%~^Kh9F(~q=n2cSn`+;(`JTn%-ahql*@oE`Io0#;h@J66jmSlGdON)Yr|O-|%&6Y( zQetElMfVgqyvX->mv5Lw)TSsaDUh;q&Ye%zeJa*jCD8HutM!v>$s99gH}+$o?zAGP ztz??JZw(DK6A95XWkLx&wk>W}nIp)P-x1@MMtbJpG?<_JMNAM!*%<9gP`>HxwvW}-D?9DM~CfCYN=W%(LL!CVeP7aO;1Tu5yh@{ z4v>6%XS!@jL0CGcKT7G$PG}j*&6`b#t0I(o^!RatZbyAl2TbgA5>Ji~C>`zX%-g#_ zK<(KC@otbdt@Rv!qT%_3n6`l!fgZ`*GB0*%&EN1=#dG6=ZetB% z8ar$qFBoA!NMpcMfo`!$$NQ!|MOwTl)ilKQleygbizV0e1hVw3n%ovFc2W#oyn8Uj z0Gn`paXxD7Ph+J#n7P8OesFn-E;oF4dXG}(N$u{o=T`MR-vc+JkI=TL>A1ake&rPx546@kIa;)$xbKkR(<1w8 z$l)sR02>okA`^Wso}VMamXn>&>W$?husnMXD+_x?Mmj2ZH5AB^zQu{Sr>X2PxpFk zp0ziR`RyKR(TrI$86#V4L}BCoyndWlavb^S#jT+ z=FEwYklAjn-S4<5Kn2e8?y*-w@2$yC=9BL_b_aJ`$fPNMdg+7m0Uwqhr+jOGDCIeW zM~bGN7lfTV;`)2x#XknV#(byF1jg8}br)CM^vTqZ!(gc_QhyJj>xwyj-tx8>`MJXQ zqhNU+`@Bk<{QE+`m(Xx$J*FGR^?juyErDF^GbY75Rcd#l87B8iw-v3+389l9^%@)X zpXjSy{P-h>q{6K$34VsUjJ4Jooc^4V#|Xw_%dY#rthLG&HMZ&`sBc45hZ52`R!2i`u5aKf7h>%LRfaK1_@5fEp~y1iiMNFSpsA( z6XKz?JwO>zl1FlGNkT^IQGMfsVq%#lNh)M=i-Xscdd;9tC#z0IBYWWN{TQK^*Ogp( zB;KyP@(Xhg(?_#9Vr~c2E15w{1?s*VQ)&oNNQ=7&GE6}}SJCQVI?s%FCAB%s3Nr$C z8~;2i_F?HU{RDyTf-HF4ykb}9bgPeAJQ;^BYI9eWqr!cPUW?a|6qEE?r(qT{j=8WW zRaSmVpgnE(!J1+eWA2-}huhlxKKuS+>$yz$vK%tH^6i-T3;B4oX2<2qcInw`f3YQw z&248hM<97_dPSaoq1t72@qNTdZgIN$`L_{;Ce~HIIbU=&EYaR{LSlBq-S)#l3s#iB z3Y)1aN{ozms|^$&o@ty46Xmudm?L$=%qUnh}~MP^8m%o&85#(B3kqd1T!} zvDX@gvZ&aJR|5c|C$+Ih79T4;hb)js=BU*Ju@xztbS~SDUn@uvKYaIvb-Q@v`X4J# zDP?S?Q}15P{i9{yy&X>qKQ7r3*Adg`5?c7{!|k^?2u|&c_3fS;`sb1FmHVfV|V+!Z{#^|~0~LJU!>5C7bU3nSsmom%{DYQ3h5C-Uppjojy^wQtyPVPjQDdx|*S z>_)5Fph(v(ai#B7_M!b%C0f$NB;2eHU}xP2jo*8Uz0Wksbxb(*9x-l)$>(A@h%Ti6 zDBfBWuaB8(-H2-qOt};q;JFf*KI(rXo8K}r0l^J-z7bREoZ1}|w%Y#4KZC=Az@h2srb3GE%eF~iVME9i+ki5 zm@eeXPz~h2@!GZzW==bxod5HwC5~Z}mpGbU54(QQ@QU*N!otF@ZcuydNsv5tZu$@Q zGtiQKuJSrt@i%?}~N|BTY-dI(>w+X1Br$qPE1!%33EtKq>Y5e(-GemD&btQos$? z+jQX?6z#Y#I2P_Md2(l0O%kK}>;~^`1v?7DD$nBv&wiIh; z{K_pSUsZ1+^xKYh!wPQYNG@2--piB9UAJK`Mm3idrv)1Hl*MQkg-!8M+2z)yN!U|h zkhQn)gki8Xb8yXQ_`@xT()G*K&^kpbmo-;kjc?Pf8*#bI>nAPb8+V9QWFi^S?^F z_OO`qw*RxDY*g%9q)8F7=n+ZHo0VQ;schfOozfue$`mJk^c${SI!`_KH1l<^oV?LnT=_y*38n5g=iFxDDCy~tL!L>$ zK4a0_tqIK4Mr-7ps`6eoZ^O=U1z9dVwj#gf1=-S^(Y-cRweoF>s`%{3Z{=(m3|#GUle)3<%h(8FK=|^%!TjD?58& zGE0Ffpi}-puI5`KW)B>(Apfv_Sui>_Y3*o#>`>dOSSUR_fU!w?N)lw!9qYi|2@cFVU%5RFT-XAa#%)d%Cl zbx#sd7}_`n-wQK>o&kjO5+8Rlh&r-@7qIgNun_p$FZ#dgcXxC14`YCA7@@j&Uh#-! zoLB92&?WcSYl#FUn)A2F-O}a8{FMTV&#b@SECt_8Yns1{LD0S6yK(RfcI(j4!LCFw zZlvC~syaUokq`Y%dJ%8okg)LDw^PO#rv3 zVdeEB@;U?5hwr=ieWL?iRE;rts0DHG#bBq3QZH%NL--6llm$Du4ITs3!FWHFvQK7c6$>QwBuSm9^nzKD9TnhKr8SZWK~szzPgei51yMz{|m{DI?G!Bghd0y@n72J4_saqdRfm*m=l05 z9Tah+XrR0(#^&)DsD{Rx>lnUFE6ZiPtmcljPu4!CR5;gz#mJl0>)>+8YDH5?|DCCq z!5keqeLWz2hi1EYl+=5MHSZPu=(FH#q-OJw^MQ~|n~ZUXcl)JrQLY)Bs%@o^j=1)( zI&-8i`-iPt=|HhG%bzvu_gpvk(2#BzJ!L&QdYfQ6c#FBAHOu|N{OH%ipS$_p4rmR7 zokp+L;re}nOW-oq!xVmqE?+IXH(56)J39*UH89!A*iyMeHcFBl0jCNT3!8-jO=Cx| zdiG)79Cc@g&^5a1YtC2x{P<~r+fPt$!wDW_dqh3H=)4)Hc`=-f08|TKa<=2gBNc2u$2mLko(?bKyea&aU#SLQ_u)-LWq`%pV{Taw=@p4<4vDtOS&upt$frQ1WPjVLSqub6U(9z9m|;|5{ccl|e%4I939C z$tufeuBvTA#L|$J>Qx|51h^Bb?RW)(M&Zok?2^6f5@a#H z_h{t!aLrP>Fgm*?=GE@q9v<(euRoWN4vfbo-9ZIR%Dpt|c4T6rkxgxznGW0#{ax}A z5?>g#)8!=F*Qmww>opwj4%~zC*#l`fI5%>88o)iV(`bW zR|0AOwTWMcyWE?0_n z*s&_kD{T>b-IDjxr0IO!rzb7QGS3J0y$gBe(ORq5{RyI#!p1bj58}3HNNS+D_De8w zLC9Cty(v4Pswr70mRL-Dz(D2!(mx zS8tF1eUbDR9?=0{Q4>e9k2W{H1Ya(i zUEZ6SBK*$ar$_^tyU4I@c3!!Dr>P#}G)jOt0HE`Zr5}G7zLPapo7mh4p%&fkPz7#P zFS>0&4t#|+J|xhoG02LUzbA%V_d;lV3cF)WJuLE2LexVQZiL9Kn$Gb%Jn^P_IAvw!Bsrs`rZfVb317+oX(d}ODyp_BUa_0GjK5ARAsy%- z4ONrkypkc5jgccga?3`SPCWBCwbi-c@!&|Jzs)vYqQdgNHg-w{LxRa>mUhP(r^-EfesMZ}D_ z)6qTc8lruoWYHs$T;vu~Byshasq@8na|awkpF;(ch?mQvJA%XwWT7WgZ9vcwsz!k) z*=58mTDF6(L8s7H)1B#i8k4(XnD0*p>w)bl&}UdxGK@w$ue9?zQ;Oi!Ndmwk(H3eU!Ww2pVm@`RYwu;BJ*!Z zk~5=Y#s3P3RKyS$GXEk$b^%QP_GN5*RD`;fm~2+F=cM9}2en@|1KkLA<{(RMruN(0;6ortX3#Tl_Ls8%!t~^bID$W;^G*aQ&*&e3`N( z!+$*!Qu1kcDh8GyPeu@rlFB5mm~N8&x{BSZYhPX9e<=Kk>7RT8$5Khq$v=7g5AnmQ z$B;WyrR^Q#WB*{vIW|GPcxCEz;3e3&dfC zSHKlO5T>q<5gq9z3n&`kCTsm3qzwddHtYZKdilE_d`VwX2G?+SaVdduL3`4w)?S&= zgB5Ex{pAm*I0y`i6a5Ys@UBho6w7e2`zOt`h^hs#HvGd|{jRXZE>V-nra}~#O(-%B zd(ZzEfr&F2-NTP^U^LlZTsI zBuL_m3;Xhl$7hBm_k^UE*=ocEH#p8N%C*BnRPwfclBa#=f!7u0=O3Q9=Al7NDb*Ai zOv-9OIy_s&{Rj6;N@8_lsUhP3{rS0lxD1wsB64Ck*zLR8i)@ZkTTr3x&pODGIP85l zZEEKx&(SUYab7G#|Bv@me~l8IXsQxft<_klHbLB5LE>#cO`42;XteE=f+-&ua8dX> zsuiy19Do>9E;W;SOR?v{6CM5{w)?Cq)8wx1m81}WkYQ3~9VTXn0CU0o_ZH;~RJ#zw zOE!j%2*IQdAm75m!WelOKq_P4yIek;JBjp2%)ws2=aC+?h#-jNJMOb;>+-*MGPXtq z@!F{s@Mzaa$m>DonCCF6BZuJ8q|A_c)=~*#=Lyuv(h&TDBQqQQa{-6prKXsWl%B{Z z`N66sN)Y{4ktti?B_v*<8zyeJBEZ|<0(73^`NqBasSY1XnC-9U+-()-HT3h|@%{P6 zTYsq%ah4fsJ91PF2U9L?HYSLOz>l#b5Np9{Lwet}EUwY$%i4DD7MI^(hE^+5oQgE} z*A7*|9+E_SQ>M}22@mp*X{T;%+})D8ks#_r4gyy|()nX#o+OzyhZIk=q^hMn^id++ zSN!W4sGj0qkzYTdJFblN>2f!;=>d{$d#<| zm;&YRp+#e5cI5Jnps$de6Cx{$oXpR=PCO$2gi>RHHBJ9>;z46l8S(=G9c66+HK(Z1 ze63$X^S^;9LEX2q61tW+srRwx5KonU`G5TSiSEr6UE|&pJf$qT39~@4t2Y?wcFyy<-0x>g>$) diff --git a/pkg/planner/actuators/platform/rdt_sequence.puml b/pkg/planner/actuators/platform/rdt_sequence.puml deleted file mode 100644 index 2d7bedc..0000000 --- a/pkg/planner/actuators/platform/rdt_sequence.puml +++ /dev/null @@ -1,69 +0,0 @@ -@startuml -'https://plantuml.com/sequence-diagram -header - RDT actuator. -endheader - -skinparam monochrome true - -actor Planner -activate Planner -participant "RDT actuator" as rdt -activate rdt -participant "Prediction script" as pred -participant "Analytics script" as analytics -database "knowledge base" as kb -activate kb -database "TS DB" as tsdb -activate tsdb -box "node level" - participant "RDT agent" as rdt_agent - activate rdt_agent -end box - -== NextState() == - -Planner -> rdt: NextState(state) -rdt -> rdt: create follow-up states. -loop for all options - note over rdt: options is a list of potential\nRDT/DRC|CAT configs\n(incl their mask settings etc.) - rdt -> pred: run - activate pred - pred -> kb: get model - kb --> pred - pred --> rdt: predicted target value. - deactivate pred -end loop -note over rdt: select potential follow-up state with\n"shortest" distance to desired state. -rdt --> Planner: [state], [utility], [action] - -== Perform() == - -Planner -> rdt: Perform(state) -rdt -> rdt: annotate POD spec with selected option. -note over rdt: TODO: add hints for scheduler.\ne.g. node level LLC miss rate < value;\n# of cache_ways/CLOSes;\n... - -loop "forever" - rdt_agent -> rdt_agent: monitor PODs on node. - alt config changed: - rdt_agent -> rdt_agent: change resctrl / taskset / etc.\naccording to config option. - note over rdt_agent: This will support DRC & e.g. RDT CAT. - else - rdt_agent -> rdt_agent: timeout. - end alt -end loop - -== Effect() == - -Planner -> rdt: Effect() -rdt -> analytics: run -activate analytics -analytics -> kb: get model -analytics -> tsdb: get data -kb --> analytics -tsdb --> analytics -analytics -> analytics: run analytics magic. -analytics --> kb: update model -deactivate analytics - -@enduml \ No newline at end of file diff --git a/pkg/planner/actuators/platform/rdt_test.go b/pkg/planner/actuators/platform/rdt_test.go index ce21034..8702294 100644 --- a/pkg/planner/actuators/platform/rdt_test.go +++ b/pkg/planner/actuators/platform/rdt_test.go @@ -1,13 +1,12 @@ package platform import ( - "os/exec" "testing" - "time" "github.com/intel/intent-driven-orchestration/pkg/common" "github.com/intel/intent-driven-orchestration/pkg/planner" + appsV1 "k8s.io/api/apps/v1" coreV1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -43,21 +42,20 @@ func (f *rdtActuatorFixture) newRdtTestActuator() *RdtActuator { f.client = fake.NewSimpleClientset(f.objects...) cfg := RdtConfig{ Interpreter: "python3", - Analytics: "test_analyze.py", - Prediction: "test_predict.py", + Analytics: "analytics/test_analyze.py", + Prediction: "analytics/test_predict.py", Options: []string{"None", "option_a", "option_b", "option_c"}, } actuator := NewRdtActuator(f.client, dummyTracer{}, cfg) + return actuator +} - cmd := exec.Command(actuator.config.Interpreter, actuator.config.Prediction) //#nosec G204 -- NA - err := cmd.Start() +func (f *rdtActuatorFixture) cleanUp(actuator *RdtActuator) { + klog.Infof("Going to kill process with PID: %d.", actuator.Cmd.Process.Pid) + err := actuator.Cmd.Process.Kill() if err != nil { - klog.Errorf("Could not start the prediction script: %s.", err) + klog.Fatalf("Could not terminate prediction service: %s", err) } - // looks like we need this on slower boxes... - time.Sleep(500 * time.Millisecond) - - return actuator } // Tests for success. @@ -66,15 +64,10 @@ func (f *rdtActuatorFixture) newRdtTestActuator() *RdtActuator { func TestRdtNextStateForSuccess(_ *testing.T) { f := newRdtActuatorFixture() actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/function-intents", Priority: 1.0, TargetKey: "default/my-function", @@ -106,6 +99,7 @@ func TestRdtPerformForSuccess(_ *testing.T) { }, } actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ Intent: common.Intent{TargetKey: "default/my-function", TargetKind: "Deployment"}, CurrentPods: map[string]common.PodState{"pod_0": {}}, @@ -118,6 +112,7 @@ func TestRdtPerformForSuccess(_ *testing.T) { func TestRdtEffectForSuccess(_ *testing.T) { f := newRdtActuatorFixture() actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ Intent: common.Intent{ TargetKey: "default/my-function", @@ -142,9 +137,11 @@ func TestRdtPerformForFailure(t *testing.T) { f := newRdtActuatorFixture() f.objects = []runtime.Object{} actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ Intent: common.Intent{ - TargetKey: "default/my-function", + TargetKey: "default/my-function", + TargetKind: "Deployment", }, CurrentPods: map[string]common.PodState{ "pod_0": {}, @@ -159,6 +156,14 @@ func TestRdtPerformForFailure(t *testing.T) { if len(f.client.Actions()) != 1 { t.Errorf("This is not expected: %v", f.client.Actions()) } + + // test same for replicaset + f.client.ClearActions() + state.Intent.TargetKind = "ReplicaSet" + actuator.Perform(&state, plan) + if len(f.client.Actions()) != 1 { + t.Errorf("This is not expected: %v", f.client.Actions()) + } } // TestRdtEffectForFailure tests for failure. @@ -166,13 +171,7 @@ func TestRdtEffectForFailure(_ *testing.T) { f := newRdtActuatorFixture() // not much to do here, as this will "just" trigger a python script. state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -186,6 +185,7 @@ func TestRdtEffectForFailure(_ *testing.T) { "p99": {ProfileType: common.ProfileTypeFromText("latency")}, } actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) actuator.config.Analytics = "foobar" // will cause a logging warning. actuator.Effect(&state, profiles) @@ -193,19 +193,56 @@ func TestRdtEffectForFailure(_ *testing.T) { // Tests for sanity. +func TestGetResourcesForSanity(t *testing.T) { + var tests = []struct { + name string + input map[string]int64 + output int64 + }{ + { + name: "should_work", + input: map[string]int64{"0_cpu_limits": 1000}, + output: 1000, + }, + { + name: "only_requests", + input: map[string]int64{"0_cpu_requests": 1000}, + output: 0, + }, + { + name: "multiple_containers", + input: map[string]int64{"1_cpu_limits": 1000, "2_cpu_limits": 2000, "0_cpu_limits": 500}, + output: 2000, + }, + { + name: "no_cpu_info", + input: map[string]int64{"0_gpu_limits": 1}, + output: 0, + }, + { + name: "faulty format", + input: map[string]int64{"a_b_c": 1}, + output: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val := getResources(tt.input) + if val != tt.output { + t.Errorf("Expected %d - got %d for test %s.", tt.output, val, tt.name) + } + }) + } +} + // TestRdtNextStateForSanity tests for sanity. func TestRdtNextStateForSanity(t *testing.T) { f := newRdtActuatorFixture() actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/function-intents", Priority: 1.0, TargetKey: "default/my-function", @@ -216,7 +253,9 @@ func TestRdtNextStateForSanity(t *testing.T) { }, }, CurrentPods: map[string]common.PodState{ - "pod_0": {}, + "pod_0": { + QoSClass: "Guaranteed", + }, }, CurrentData: map[string]map[string]float64{ "cpu_value": {"node": 10.0}, @@ -230,8 +269,9 @@ func TestRdtNextStateForSanity(t *testing.T) { "default/blurb": 10.0, } profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/blurb": {ProfileType: common.ProfileTypeFromText("availability")}, + "default/p95": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/blurb": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, } states, utils, actions := actuator.NextState(&state, &goal, profiles) @@ -242,7 +282,7 @@ func TestRdtNextStateForSanity(t *testing.T) { if _, found := states[0].Annotations["rdtVisited"]; !found { t.Errorf("Expected the temp blocker in annotation: %v.", states[0].Annotations) } - // check if resulting action match. + // check if the resulting action matches. if actions[0].Name != actuator.Name() || actions[0].Properties.(map[string]string)["option"] != "option_b" { t.Errorf("Expected a action to set option_b - got: %v.", actions[0]) } @@ -259,6 +299,7 @@ func TestRdtNextStateForSanity(t *testing.T) { } // expect empty res if we are good for now. + state.Annotations[actuator.config.AnnotationName] = "option_b" state.Intent.Objectives["default/p99"] = 4.9 goal.Intent.Objectives["default/p99"] = 5.0 _, _, actions = actuator.NextState(&state, &goal, profiles) @@ -286,15 +327,45 @@ func TestRdtNextStateForSanity(t *testing.T) { } // we do not expect to set an option if it is already set... - state.Annotations["configureRDT"] = "option_a" - delete(state.Annotations, "rdt_visited") + state.Annotations[actuator.config.AnnotationName] = "option_a" + delete(state.Annotations, "rdtVisited") _, _, actions = actuator.NextState(&state, &goal, profiles) if len(actions) != 0 { t.Errorf("Expected no actions - got: %v.", actions) } - // expect empty is script return -1.0 + // expect empty result if script return -1.0 + delete(state.Annotations, "rdtVisited") + delete(state.Annotations, actuator.config.AnnotationName) + delete(state.Intent.Objectives, "default/p99") + delete(goal.Intent.Objectives, "default/p99") state.Intent.Objectives["default/p72"] = 10.0 + goal.Intent.Objectives["default/p72"] = 3.0 + _, _, actions = actuator.NextState(&state, &goal, profiles) + if len(actions) != 0 { + t.Errorf("Expected no actions - got: %v.", actions) + } + + // check if we have multiple results when we have multiple targeted objectives. + delete(state.Intent.Objectives, "default/p72") + delete(state.Annotations, "rdtVisited") + delete(state.Annotations, actuator.config.AnnotationName) + state.Intent.Objectives["default/p99"] = 20 + state.Intent.Objectives["default/p95"] = 15 + goal.Intent.Objectives["default/p99"] = 5.0 + goal.Intent.Objectives["default/p95"] = 7.5 + _, _, actions = actuator.NextState(&state, &goal, profiles) + if len(actions) != 2 { + t.Errorf("Expected 2 actions - got: %v.", actions) + } + + // POD not in guaranteed class. + delete(state.Annotations, "rdtVisited") + delete(state.Intent.Objectives, "default/p72") + delete(state.CurrentPods, "pod_0") + state.CurrentPods["pod_1"] = common.PodState{ + QoSClass: "Burstable", + } _, _, actions = actuator.NextState(&state, &goal, profiles) if len(actions) != 0 { t.Errorf("Expected no actions - got: %v.", actions) @@ -307,15 +378,10 @@ func TestRdtNextStateForSanity(t *testing.T) { func BenchmarkNextState(b *testing.B) { f := newRdtActuatorFixture() actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/function-intents", Priority: 1.0, TargetKey: "default/my-function", @@ -326,7 +392,9 @@ func BenchmarkNextState(b *testing.B) { }, }, CurrentPods: map[string]common.PodState{ - "pod_0": {}, + "pod_0": { + QoSClass: "Guaranteed", + }, }, CurrentData: map[string]map[string]float64{ "cpu_value": {"node": 10.0}, @@ -340,8 +408,8 @@ func BenchmarkNextState(b *testing.B) { "default/blurb": 10.0, } profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/blurb": {ProfileType: common.ProfileTypeFromText("availability")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/blurb": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, } b.ResetTimer() @@ -363,11 +431,25 @@ func TestRdtPerformForSanity(t *testing.T) { Namespace: "default", }, }, + &appsV1.Deployment{ + ObjectMeta: metaV1.ObjectMeta{ + Name: "my-function", + Namespace: "default", + }, + }, + &appsV1.ReplicaSet{ + ObjectMeta: metaV1.ObjectMeta{ + Name: "my-function", + Namespace: "default", + }, + }, } actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) state := common.State{ Intent: common.Intent{ - TargetKey: "default/my-function", + TargetKey: "default/my-function", + TargetKind: "Deployment", }, CurrentPods: map[string]common.PodState{ "pod_0": {}, @@ -410,6 +492,43 @@ func TestRdtPerformForSanity(t *testing.T) { if j != len(expectedActions) { t.Errorf("Expected more actions: %v - found only: %d.", expectedActions, j) } + + // check if actions are also called for replicaset. + f.client.ClearActions() + state.Intent.TargetKind = "ReplicaSet" + plan = []planner.Action{ + {Name: rdtActionName, Properties: map[string]string{"option": "option_a"}}, + } + actuator.Perform(&state, plan) + expectedActions = []string{"get", "update"} + j = 0 + for i, action := range f.client.Actions() { + if action.GetVerb() != expectedActions[i] { + t.Errorf("Expected %s - got %s.", expectedActions[i], action) + } + j++ + } + if j != len(expectedActions) { + t.Errorf("Expected more actions: %v - found only: %d.", expectedActions, j) + } + + // check if annotation is removed if option is None: + f.client.ClearActions() + plan = []planner.Action{ + {Name: rdtActionName, Properties: map[string]string{"option": "None"}}, + } + actuator.Perform(&state, plan) + expectedActions = []string{"get", "update"} + j = 0 + for i, action := range f.client.Actions() { + if action.GetVerb() != expectedActions[i] { + t.Errorf("Expected %s - got %s.", expectedActions[i], action) + } + j++ + } + if j != len(expectedActions) { + t.Errorf("Expected more actions: %v - found only: %d.", expectedActions, j) + } } // TestRdtEffectForSanity tests for sanity. @@ -417,13 +536,7 @@ func TestRdtEffectForSanity(_ *testing.T) { f := newRdtActuatorFixture() // not much to do here, as this will "just" trigger a python script. state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -435,10 +548,11 @@ func TestRdtEffectForSanity(_ *testing.T) { }, } profiles := map[string]common.Profile{ - "p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "avail": {ProfileType: common.ProfileTypeFromText("availability")}, + "p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "avail": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, } actuator := f.newRdtTestActuator() + defer f.cleanUp(actuator) // will cause a logging warning. actuator.Effect(&state, profiles) } diff --git a/pkg/planner/actuators/scaling/analytics/cpu_rightsizing.py b/pkg/planner/actuators/scaling/analytics/cpu_rightsizing.py index 2c563a5..0973495 100644 --- a/pkg/planner/actuators/scaling/analytics/cpu_rightsizing.py +++ b/pkg/planner/actuators/scaling/analytics/cpu_rightsizing.py @@ -13,6 +13,7 @@ import io import logging import os +from datetime import datetime, timezone import matplotlib.pyplot as plt import numpy as np @@ -115,13 +116,13 @@ def store_result(popt, latency_range = (min(data[args.latency]), max(data[args.latency])) cpu_range = (min(data["cpus"]), max(data["cpus"])) training_features = ["cpus"] - timestamp = datetime.datetime.utcnow() + timestamp = datetime.now(timezone.utc) doc = {"name": args.name, "profileName": args.latency, "group": "vertical_scaling", "data": {"latencyRange": latency_range, "cpuRange": cpu_range, - "popt": popt.tolist(), + "popt": [popt.tolist()], "trainingFeatures": training_features, "targetFeature": args.latency, "image": img}, diff --git a/pkg/planner/actuators/scaling/cpu_scale.go b/pkg/planner/actuators/scaling/cpu_scale.go index 5ad4f4b..a47f66b 100644 --- a/pkg/planner/actuators/scaling/cpu_scale.go +++ b/pkg/planner/actuators/scaling/cpu_scale.go @@ -25,9 +25,6 @@ import ( // delimiter for the key entries in the resources hashmap. const delimiter = "_" -// boostFactor defines the multiplication factor for calculating resource limits from requests. -const boostFactor = 1.0 - // actionName represents the name of the action. const actionName = "scaleCPU" @@ -42,6 +39,7 @@ type CPUScaleConfig struct { CPUMax int64 `json:"cpu_max"` CPURounding int64 `json:"cpu_rounding"` CPUSafeGuardFactor float64 `json:"cpu_safeguard_factor"` + BoostFactor float64 `json:"boost_factor"` MaxProActiveCPU int64 `json:"max_proactive_cpu"` ProActiveLatencyPercentage float64 `json:"proactive_latency_percentage"` LookBack int `json:"look_back"` @@ -57,7 +55,7 @@ type CPUScaleEffect struct { // Never ever think about making these non-public! Needed for marshalling this struct. LatencyRange [2]float64 CPURange [2]float64 - Popt [3]float64 + Popts [][3]float64 TrainingFeatures [1]string TargetFeature string Image string @@ -82,37 +80,49 @@ func (cs CPUScaleActuator) Group() string { // predictLatency uses the knowledge base to calculate the latency. It does use the parameters popt // that are obtained when sum of the squared residuals which is minimized. More info in: // https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html -func (cs CPUScaleActuator) predictLatency(popt []float64, limCPU float64) float64 { +func (cs CPUScaleActuator) predictLatency(popt [3]float64, limCPU float64) float64 { result := popt[0]*math.Exp(-popt[1]*(limCPU/1000)) + popt[2] return result } // getResourceValues return the cpu resources associated with the last container of a POD. -func getResourceValues(state *common.State) int64 { - cpuLimit := int64(0) +func getResourceValues(state *common.State) (int64, int) { cpuRequest := int64(0) + cpuLimit := int64(0) lastIndex := -1 for key, value := range state.Resources { items := strings.Split(key, delimiter) + if len(items) != 3 { + continue + } index, err := strconv.Atoi(items[0]) if err != nil { - klog.Errorf("Failed to convert: %v", err) - return 0 + klog.Errorf("Failed to convert index: %v", err) + continue + } + if items[1] != "cpu" { + continue + } + if index > lastIndex { + cpuRequest = 0 + cpuLimit = 0 + lastIndex = index } - if items[1] == "cpu" && index >= lastIndex { + if index == lastIndex { if items[2] == "requests" { cpuRequest = value } else if items[2] == "limits" { cpuLimit = value - cpuRequest = value } - lastIndex = index } } + if lastIndex == -1 { + return 0, -1 + } if cpuLimit >= cpuRequest { - return cpuLimit + return cpuLimit, lastIndex } - return cpuRequest + return cpuRequest, lastIndex } // setResourceValues tweaks the resource requests limits on the workload. @@ -120,9 +130,6 @@ func (cs CPUScaleActuator) setResourceValues(state *common.State, newValue int) tmp := strings.Split(state.Intent.TargetKey, "/") namespace := tmp[0] - // TODO: once is it configurable: if boost factor is misconfigured, display warning and reset to 1.0. - factor := boostFactor - if state.Intent.TargetKind == "Deployment" { client := cs.apps.AppsV1().Deployments(namespace) retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -139,11 +146,13 @@ func (cs CPUScaleActuator) setResourceValues(state *common.State, newValue int) } updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Requests["cpu"] = request - limit := resource.NewMilliQuantity(int64(float64(newValue)*factor), resource.DecimalSI).DeepCopy() - if len(updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Limits) == 0 { - updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Limits = make(map[v1.ResourceName]resource.Quantity) + if cs.cfg.BoostFactor >= 1.0 { + limit := resource.NewMilliQuantity(int64(float64(newValue)*cs.cfg.BoostFactor), resource.DecimalSI).DeepCopy() + if len(updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Limits) == 0 { + updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Limits = make(map[v1.ResourceName]resource.Quantity) + } + updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Limits["cpu"] = limit } - updatedDeployment.Spec.Template.Spec.Containers[len(updatedDeployment.Spec.Template.Spec.Containers)-1].Resources.Limits["cpu"] = limit _, updateErr := client.Update(context.TODO(), updatedDeployment, metaV1.UpdateOptions{}) return updateErr @@ -167,11 +176,13 @@ func (cs CPUScaleActuator) setResourceValues(state *common.State, newValue int) } updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Requests["cpu"] = request - limit := resource.NewMilliQuantity(int64(float64(newValue)*factor), resource.DecimalSI).DeepCopy() - if len(updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Limits) == 0 { - updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Limits = make(map[v1.ResourceName]resource.Quantity) + if cs.cfg.BoostFactor >= 1.0 { + limit := resource.NewMilliQuantity(int64(float64(newValue)*cs.cfg.BoostFactor), resource.DecimalSI).DeepCopy() + if len(updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Limits) == 0 { + updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Limits = make(map[v1.ResourceName]resource.Quantity) + } + updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Limits["cpu"] = limit } - updatedReplicaSet.Spec.Template.Spec.Containers[len(updatedReplicaSet.Spec.Template.Spec.Containers)-1].Resources.Limits["cpu"] = limit _, updateErr := client.Update(context.TODO(), updatedReplicaSet, metaV1.UpdateOptions{}) return updateErr @@ -189,57 +200,104 @@ func roundUpCores(n int64, fraction int64) int64 { return b } -// findState tries to determine the best possible # of cpus. -func (cs CPUScaleActuator) findState( - state *common.State, - goal *common.State, - currentCPU int64, - profiles map[string]common.Profile) (common.State, int64, error) { - - newState := state.DeepCopy() - newCPUValue := int64(0) - for k := range state.Intent.Objectives { - if profiles[k].ProfileType == common.ProfileTypeFromText("latency") { - res, err := cs.tracer.GetEffect(state.Intent.Key, cs.Group(), k, cs.cfg.LookBack, func() interface{} { +// getScalingEffects return all entries in the knowledgebase for latency related objectives. +func (cs CPUScaleActuator) getScalingEffects(intent common.Intent, profiles map[string]common.Profile) (map[string]*CPUScaleEffect, error) { + result := map[string]*CPUScaleEffect{} + for key := range intent.Objectives { + if profiles[key].ProfileType == common.ProfileTypeFromText("latency") { + res, err := cs.tracer.GetEffect(intent.Key, cs.Group(), key, cs.cfg.LookBack, func() interface{} { return &CPUScaleEffect{} }) if res == nil || err != nil { - return common.State{}, 0, fmt.Errorf("could not retrieve information from knowledge base for: %s", state.Intent.TargetKey) - } - if goal.Intent.Objectives[k] < res.(*CPUScaleEffect).Popt[2] { - return common.State{}, 0, fmt.Errorf("the model cannot handle this case - aborting for: %s", state.Intent.TargetKey) - } - latency := goal.Intent.Objectives[k] * cs.cfg.CPUSafeGuardFactor // TODO: config file. - newState.Intent.Objectives[k] = latency - popt := res.(*CPUScaleEffect).Popt - cpuValue := int64(-(1 / popt[1]) * math.Log((latency-popt[2])/popt[0]) * 1000) - if cpuValue > newCPUValue { - newCPUValue = roundUpCores(cpuValue, cs.cfg.CPURounding) + return nil, fmt.Errorf("could not retrieve information from knowledge base for: %s", key) } + result[key] = res.(*CPUScaleEffect) } } - if newState.IsBetter(goal, profiles) && newCPUValue != currentCPU { - index := -1 - for key := range newState.Resources { - items := strings.Split(key, delimiter) - tmp, err := strconv.Atoi(items[0]) - if err != nil { - klog.Errorf("Failed to convert to int: %v", err) + return result, nil +} + +// findStates tries to determine the best possible state for each latency related objective. +func (cs CPUScaleActuator) findStates( + state *common.State, + goal *common.State, + currentCPU int64, + containerIndex int, + profiles map[string]common.Profile) ([]common.State, []float64, []planner.Action, error) { + var candidates []common.State + var utilities []float64 + var actions []planner.Action + + // get insights from the knowledge base. + effects, err := cs.getScalingEffects(state.Intent, profiles) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not retrieve information from knowledge base for: %s", state.Intent.TargetKey) + } + + // construct a set of follow-up states. + for k := range state.Intent.Objectives { + if profiles[k].ProfileType == common.ProfileTypeFromText("latency") { + newState := state.DeepCopy() + latency := goal.Intent.Objectives[k] * cs.cfg.CPUSafeGuardFactor + popts := effects[k].Popts + newCPUValue := int64(0) + for _, popt := range popts { + if goal.Intent.Objectives[k] < popt[2] { + klog.Warningf("the model cannot handle this case - aborting for: %s", state.Intent.TargetKey) + continue + } + latency := goal.Intent.Objectives[k] * cs.cfg.CPUSafeGuardFactor + newState.Intent.Objectives[k] = latency + cpuValue := int64(-(1 / popt[1]) * math.Log((latency-popt[2])/popt[0]) * 1000) + if cpuValue > newCPUValue { + newCPUValue = roundUpCores(cpuValue, cs.cfg.CPURounding) + break + } } - if tmp > index { - index = tmp + + if newCPUValue != currentCPU && newCPUValue > 0 { + // forecast the effect of vertical scaling. + for objectiveKey := range effects { + newState.Intent.Objectives[objectiveKey] = latency + } + + // resources can be nil so need a quick check here. + // setting requests equal to limits leads to a guaranteed POD QoS. + for name, pod := range newState.CurrentPods { + pod.QoSClass = "BestEffort" + if cs.cfg.BoostFactor == 1.0 { + pod.QoSClass = "Guaranteed" + } else if cs.cfg.BoostFactor > 1.0 { + pod.QoSClass = "Burstable" + } + newState.CurrentPods[name] = pod + } + + if newState.Resources == nil { + newState.Resources = make(map[string]int64) + } + newState.Resources[strings.Join([]string{strconv.Itoa(containerIndex), "cpu", "requests"}, delimiter)] = newCPUValue + if cs.cfg.BoostFactor >= 1.0 { + newState.Resources[strings.Join([]string{strconv.Itoa(containerIndex), "cpu", "limits"}, delimiter)] = int64(float64(newCPUValue) * cs.cfg.BoostFactor) + } + newState.CurrentData[cs.Name()] = map[string]float64{cs.Name(): 1} + + // utility function. + utility := float64(newCPUValue) / float64(cs.cfg.CPUMax) + if newCPUValue > currentCPU { + utility *= 1.0 / goal.Intent.Priority + } + + // the associated action. + action := planner.Action{Name: cs.Name(), Properties: map[string]int64{"value": newCPUValue}} + + candidates = append(candidates, newState) + utilities = append(utilities, utility) + actions = append(actions, action) } } - // resources can be nil so need a quick check here. - if newState.Resources == nil { - newState.Resources = make(map[string]int64) - } - newState.Resources[strings.Join([]string{strconv.Itoa(index), "cpu", "limits"}, delimiter)] = newCPUValue - newState.Resources[strings.Join([]string{strconv.Itoa(index), "cpu", "requests"}, delimiter)] = newCPUValue - newState.CurrentData[cs.Name()] = map[string]float64{cs.Name(): 1} - return newState, newCPUValue, nil } - return common.State{}, 0, nil + return candidates, utilities, actions, nil } // proactiveScaling adds a state based on hypothetical improvement on the objectives. @@ -307,26 +365,20 @@ func (cs CPUScaleActuator) NextState(state *common.State, goal *common.State, if len(state.CurrentPods) == 0 { return nil, nil, nil } - // let's find a follow-up state. - currentValue := getResourceValues(state) - newState, newValue, err := cs.findState(state, goal, currentValue, profiles) - if newValue != 0 && err == nil { - utility := float64(newValue) / float64(cs.cfg.CPUMax) - if newValue > currentValue { - utility *= 1.0 / goal.Intent.Priority - } - return []common.State{newState}, []float64{utility}, []planner.Action{ - {Name: cs.Name(), Properties: map[string]int64{"value": newValue}}, - } + // let's find follow-up states. + currentValue, containerIndex := getResourceValues(state) + newStates, utilities, actions, err := cs.findStates(state, goal, currentValue, containerIndex, profiles) + if err == nil && len(newStates) > 0 { + return newStates, utilities, actions } // if the actuator is allowed to proactively scale - let's try that. - if cs.cfg.MaxProActiveCPU > 0.0 && err != nil { + if cs.cfg.MaxProActiveCPU > 0.0 { // if no state was found or an error is returned, the proactive from NextState is actioned. klog.V(2).Infof("Proactive mode is enabled - will try to do sth for: %s.", state.Intent.TargetKey) proactiveState, proactiveUtility, proactivePlan := cs.proactiveScaling(state, goal, currentValue, profiles) return proactiveState, proactiveUtility, proactivePlan } - klog.Warningf("Could not find (better) next state for %s: %v.", state.Intent.TargetKey, err) + klog.Warningf("Could not find (better) next state for %s; err was: %v.", state.Intent.TargetKey, err) return nil, nil, nil } diff --git a/pkg/planner/actuators/scaling/cpu_scale_test.go b/pkg/planner/actuators/scaling/cpu_scale_test.go index 40905b2..7572d63 100644 --- a/pkg/planner/actuators/scaling/cpu_scale_test.go +++ b/pkg/planner/actuators/scaling/cpu_scale_test.go @@ -37,7 +37,7 @@ func (d dummyTracerCPU) GetEffect(_ string, _ string, profileName string, tmp := constructor().(*CPUScaleEffect) // the values will affect the latency and the tests results - tmp.Popt = [3]float64{400, 2, 30} + tmp.Popts = [][3]float64{{400, 2, 30}} return tmp, nil } @@ -63,6 +63,7 @@ func (f *CPUScaleActuatorFixture) newCPUScaleTestActuator(proactive bool) *CPUSc CPUMax: 2000, CPUSafeGuardFactor: 0.95, CPURounding: 100, + BoostFactor: 1.0, ProActiveLatencyPercentage: 0.8, } if proactive { @@ -122,7 +123,7 @@ func createReplicaSet() runtime.Object { Containers: []v1.Container{ {Name: "pod0123", Resources: v1.ResourceRequirements{ - Limits: map[v1.ResourceName]resource.Quantity{ + Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceCPU: resource.MustParse("200"), }, }, @@ -140,13 +141,7 @@ func createReplicaSet() runtime.Object { func TestCPUScaleNextStateForSuccess(t *testing.T) { f := newCPUScaleActuatorFixture(t) actuator := f.newCPUScaleTestActuator(false) - state := common.State{Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + state := common.State{Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -154,7 +149,7 @@ func TestCPUScaleNextStateForSuccess(t *testing.T) { Objectives: map[string]float64{"p99": 20.0}}} goal := common.State{} goal.Intent.Objectives = map[string]float64{"p99": 10.0} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} actuator.NextState(&state, &goal, profiles) } @@ -180,7 +175,7 @@ func TestCPUScaleEffectForSuccess(t *testing.T) { f := newCPUScaleActuatorFixture(t) actuator := f.newCPUScaleTestActuator(false) s0 := common.State{} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} actuator.Effect(&s0, profiles) } @@ -198,13 +193,7 @@ func TestCPUScaleNextStateForFailure(t *testing.T) { actuator := f.newCPUScaleTestActuator(false) state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -230,12 +219,12 @@ func TestCPUScaleNextStateForFailure(t *testing.T) { "default/availability": 0.999, } profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, } // no data in knowledge base. - profiles["default/throughput"] = common.Profile{ProfileType: common.ProfileTypeFromText("throughput")} - profiles["default/blurb"] = common.Profile{ProfileType: common.ProfileTypeFromText("latency")} + profiles["default/throughput"] = common.Profile{ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false} + profiles["default/blurb"] = common.Profile{ProfileType: common.ProfileTypeFromText("latency"), Minimize: true} state.Intent.Objectives["default/blurb"] = 42.0 state.Intent.Objectives["default/throughput"] = 200.0 states, _, _ := actuator.NextState(&state, &goal, profiles) @@ -320,7 +309,7 @@ func TestCPUScalePerformForFailure(t *testing.T) { // TestCPUScaleGetResourcesForFailure tests for failure func TestCPUScaleGetResourcesForFailure(t *testing.T) { s0 := common.State{Resources: map[string]int64{"a_cpu_limits": 100}} - res := getResourceValues(&s0) + res, _ := getResourceValues(&s0) if res != 0 { t.Errorf("Should have been 0 - was: %d.", res) } @@ -334,13 +323,7 @@ func TestCPUScaleNextStateForSanity(t *testing.T) { actuator := f.newCPUScaleTestActuator(false) state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -363,8 +346,9 @@ func TestCPUScaleNextStateForSanity(t *testing.T) { goal := common.State{} goal.Intent.Objectives = map[string]float64{"default/p99": 50.0} profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/p95": {ProfileType: common.ProfileTypeFromText("latency")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/p95": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/p50": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, } // if we are better than goal -> do nothing. @@ -387,7 +371,7 @@ func TestCPUScaleNextStateForSanity(t *testing.T) { delete(state.CurrentData, actionName) _, _, actions = actuator.NextState(&state, &goal, profiles) if len(actions) != 1 || actions[0].Properties.(map[string]int64)["value"] != 800 { - t.Errorf("Extpected one action to set 800 - got: %v", actions) + t.Errorf("Expected one action to set 800 - got: %v", actions) } // to strict of a goal. @@ -415,7 +399,7 @@ func TestCPUScaleNextStateForSanity(t *testing.T) { // maxProactive reached. delete(state.CurrentPods, "proactiveResourceAlloc") state.Resources = map[string]int64{ - "1_cpu_limits": actuator.cfg.MaxProActiveCPU, + "1_cpu_requests": actuator.cfg.MaxProActiveCPU, } states, utilities, actions = actuator.NextState(&state, &goal, profiles) if len(states) != 0 || len(utilities) != 0 || len(actions) != 0 { @@ -432,18 +416,27 @@ func TestCPUScaleNextStateForSanity(t *testing.T) { t.Errorf("Should contain 1 proactive action; was: %v", actions) } + // ensure we get as many states back as we have objectives. + delete(state.Intent.Objectives, "default/p95") + delete(goal.Intent.Objectives, "default/p95") + state.Resources["1_cpu_limits"] = 1600 + state.Resources["1_cpu_requests"] = 1600 + state.Intent.Objectives["default/p99"] = 50 + state.Intent.Objectives["default/p50"] = 50 + goal.Intent.Objectives["default/p99"] = 40 + goal.Intent.Objectives["default/p50"] = 40 + klog.Infof("Current %+v, Goal %+v", state, goal) + states, _, _ = actuator.NextState(&state, &goal, profiles) + if len(states) != 2 { + t.Errorf("Should have returned 2 states; was: %v", states) + } + // ensure an "empty" state does not crash the actuator. actuator = f.newCPUScaleTestActuator(false) delete(goal.Intent.Objectives, "default/p95") goal.Intent.Objectives["default/p99"] = 120 emptyState := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -455,6 +448,35 @@ func TestCPUScaleNextStateForSanity(t *testing.T) { if len(actions) != 0 { t.Errorf("Should contain no action; was: %v", actions) } + + // test boost factor - QoS should be set accordingly. + actuator = f.newCPUScaleTestActuator(false) + actuator.cfg.BoostFactor = 0.8 + delete(state.Intent.Objectives, "default/p95") + state.Resources = nil + state.Intent.Objectives["default/p99"] = 200 + states, _, _ = actuator.NextState(&state, &goal, profiles) + if len(states) != 2 { + t.Errorf("Should have return at least 2 states - was %v", states) + } + for name, pod := range states[0].CurrentPods { + if pod.QoSClass != "BestEffort" { + t.Errorf("Expected POD %s to be in besteffort QoS - was: %s.", name, pod.QoSClass) + } + } + + // boost factor >= 1.0 + actuator = f.newCPUScaleTestActuator(false) + actuator.cfg.BoostFactor = 2.0 + states, _, _ = actuator.NextState(&state, &goal, profiles) + if len(states) != 2 { // 2 objectives. + t.Errorf("Should have return at least 2 state - was %v", states) + } + for name, pod := range states[0].CurrentPods { + if pod.QoSClass != "Burstable" { + t.Errorf("Expected POD %s to be in burstable QoS - was: %s.", name, pod.QoSClass) + } + } } // TestCPUScalePerformForSanity tests for sanity. @@ -521,6 +543,7 @@ func TestCPUScalePerformForSanity(t *testing.T) { // Test boost factor. f.client.ClearActions() f.objects = []runtime.Object{createReplicaSet()} + actuator = f.newCPUScaleTestActuator(false) actuator.Perform(&s0, plan) expectedActions = []string{"get", "update"} for i, action := range f.client.Actions() { @@ -542,32 +565,40 @@ func TestCPUScalePerformForSanity(t *testing.T) { t.Errorf("Limits should have been 1000; was: %v", res[0].Resources.Limits["cpu"]) } - //// Boost factor < 1.0 should reset to 1.0 - //f.client.ClearActions() - //f.objects = []runtime.Object{createReplicaSet()} - //actuator.cfg.BoostFactor = 0.9 - //actuator.Perform(&s0, plan) - //updatedRS, _ = f.client.AppsV1().ReplicaSets("default").Get(context.TODO(), "my-replicaset", metaV1.GetOptions{}) - //res = updatedRS.Spec.Template.Spec.Containers - //val, ok = res[0].Resources.Limits["cpu"] - //if val.MilliValue() != 1000 || !ok { - // t.Errorf("Limits should have been 1000; was: %v", res[0].Resources.Limits["cpu"]) - //} + // Boost factor < 1.0 should lead to no limits being set. + f.client.ClearActions() + f.objects = []runtime.Object{createReplicaSet()} + actuator = f.newCPUScaleTestActuator(false) + actuator.cfg.BoostFactor = 0.9 + actuator.Perform(&s0, plan) + updatedRS, _ = f.client.AppsV1().ReplicaSets("default").Get(context.TODO(), "my-replicaset", metaV1.GetOptions{}) + res = updatedRS.Spec.Template.Spec.Containers + val, ok = res[0].Resources.Limits["cpu"] + if ok { + t.Errorf("Limits should not have been set; was: %v", val) + } + + // Boost factor > 1.0 should lead to POD being in Burstable QoS. + f.client.ClearActions() + f.objects = []runtime.Object{createReplicaSet()} + actuator = f.newCPUScaleTestActuator(false) + actuator.cfg.BoostFactor = 2.0 + actuator.Perform(&s0, plan) + updatedRS, _ = f.client.AppsV1().ReplicaSets("default").Get(context.TODO(), "my-replicaset", metaV1.GetOptions{}) + res = updatedRS.Spec.Template.Spec.Containers + val, ok = res[0].Resources.Limits["cpu"] + if val.MilliValue() != 2000 || !ok { + t.Errorf("Limits should have been 2000; was: %v", res[0].Resources.Limits["cpu"]) + } } // TestCPUScaleEffectForSanity tests for sanity. func TestCPUScaleEffectForSanity(t *testing.T) { f := newCPUScaleActuatorFixture(t) // this will "just" trigger a python script. - state := common.State{Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", + state := common.State{Intent: common.Intent{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0}}} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} actuator := f.newCPUScaleTestActuator(false) actuator.Effect(&state, profiles) @@ -579,27 +610,26 @@ func TestCPUScaleEffectForSanity(t *testing.T) { // TestCPUScaleGetResourcesForSuccess tests for sanity. func TestCPUScaleGetResourcesForSanity(t *testing.T) { s0 := common.State{Resources: map[string]int64{}} - res := getResourceValues(&s0) + res, _ := getResourceValues(&s0) if res != 0 { t.Errorf("Should have been 0 - was: %v", res) } - // request defined. s0.Resources["0_cpu_requests"] = 200 - res = getResourceValues(&s0) + res, _ = getResourceValues(&s0) if res != 200 { t.Errorf("Should have been 200 - was: %v", res) } // limits defined. s0.Resources["0_cpu_limits"] = 400 - res = getResourceValues(&s0) + res, _ = getResourceValues(&s0) if res != 400 { t.Errorf("Should have been 400 - was: %v", res) } // the last container matters. - s0.Resources["1_cpu_limits"] = 100 - res = getResourceValues(&s0) - if res != 100 { + s0.Resources["1_cpu_requests"] = 100 + res, containerIndex := getResourceValues(&s0) + if res != 100 || containerIndex != 1 { t.Errorf("Should have been 100 - was: %v", res) } } @@ -685,8 +715,8 @@ func TestCPUScaleActuator_NextState(t *testing.T) { }, goal: newState, profiles: map[string]common.Profile{ - "default/p95latency": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/availability": {ProfileType: common.ProfileTypeFromText("availability")}, + "default/p95latency": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/availability": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, }, }, @@ -706,7 +736,7 @@ func TestCPUScaleActuator_NextState(t *testing.T) { apps: tt.fields.apps, } newState.Intent.Objectives["default/p95latency"] = cs.predictLatency( - []float64{400, 2, 30}, 900) + [3]float64{400, 2, 30}, 900) newState.Intent.Objectives["default/availability"] = 1 got, got1, got2 := cs.NextState(&tt.args.state, &tt.args.goal, tt.args.profiles) //#nosec G601 -- NA as this is a test. @@ -757,7 +787,7 @@ func TestCPUScaleActuator_predictLatencyCPU(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := actuator.predictLatency(tt.args.popt, float64(tt.args.limCPU)); math.Round(got) != tt.want { + if got := actuator.predictLatency([3]float64(tt.args.popt), float64(tt.args.limCPU)); math.Round(got) != tt.want { t.Errorf("predictLatency() = %v, want %v", got, tt.want) } }) diff --git a/pkg/planner/actuators/scaling/rm_pod_test.go b/pkg/planner/actuators/scaling/rm_pod_test.go index 5751285..5640096 100644 --- a/pkg/planner/actuators/scaling/rm_pod_test.go +++ b/pkg/planner/actuators/scaling/rm_pod_test.go @@ -43,13 +43,7 @@ func (f *rmPodActuatorFixture) newRmPodTestActuator() *RmPodActuator { func TestRmNextStateForSuccess(t *testing.T) { f := newRmPodActuatorFixture(t) start := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -64,8 +58,8 @@ func TestRmNextStateForSuccess(t *testing.T) { } goal := common.State{} profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/rps": {ProfileType: common.ProfileTypeFromText("throughput")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/rps": {ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false}, } actuator := f.newRmPodTestActuator() actuator.NextState(&start, &goal, profiles) @@ -83,13 +77,7 @@ func TestRmPerformForSuccess(t *testing.T) { }, } s0 := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "my-deployment", Priority: 1.0, TargetKey: "default/my-deployment", @@ -120,13 +108,7 @@ func TestRmNextStateForFailure(t *testing.T) { actuator := f.newRmPodTestActuator() state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -140,7 +122,7 @@ func TestRmNextStateForFailure(t *testing.T) { goal := common.State{} goal.Intent.Objectives = map[string]float64{"default/p99": 3.0, "default/rps": 0.0, "default/availability": 0.999} profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, } // no throughput is being tracked. @@ -165,13 +147,7 @@ func TestRmPerformForFailure(t *testing.T) { f := newRmPodActuatorFixture(t) f.objects = []runtime.Object{} s0 := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "my-deployment", Priority: 1.0, TargetKey: "default/my-deployment", @@ -195,13 +171,7 @@ func TestRmPerformForFailure(t *testing.T) { func TestRmNextStateForSanity(t *testing.T) { f := newRmPodActuatorFixture(t) start := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -220,9 +190,9 @@ func TestRmNextStateForSanity(t *testing.T) { goal := common.State{} goal.Intent.Priority = 1.0 profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/rps": {ProfileType: common.ProfileTypeFromText("throughput")}, - "default/availability": {ProfileType: common.ProfileTypeFromText("availability")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/rps": {ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false}, + "default/availability": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, } actuator := f.newRmPodTestActuator() states, utilities, actions := actuator.NextState(&start, &goal, profiles) @@ -249,13 +219,7 @@ func TestRmPerformForSanity(t *testing.T) { }, } s0 := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "my-deployment", Priority: 1.0, TargetKey: "default/my-deployment", diff --git a/pkg/planner/actuators/scaling/scale_out.go b/pkg/planner/actuators/scaling/scale_out.go index a16a089..8efc0bd 100644 --- a/pkg/planner/actuators/scaling/scale_out.go +++ b/pkg/planner/actuators/scaling/scale_out.go @@ -47,7 +47,6 @@ type ScaleOutConfig struct { // ScaleOutEffect describes the data that is stored in the knowledge base. type ScaleOutEffect struct { - // TODO: make private again once refactored. // Never ever think about making these non-public! Needed for marshalling this struct. ThroughputRange [2]float64 ThroughputScale [2]float64 @@ -93,10 +92,14 @@ func predictLatency(popt [4]float64, throughput float64, numPods int) float64 { return (popt[0] * math.Exp(popt[1]*throughput)) / (popt[2] * math.Exp(popt[3]*throughput*float64(numPods))) } -// findState tries to determine the best possible # of replicas. -func (scale ScaleOutActuator) findState(state *common.State, goal *common.State, throughputObjective string, profiles map[string]common.Profile) (common.State, error) { +// findStates tries to determine the best possible # of replicas. +func (scale ScaleOutActuator) findStates(state *common.State, goal *common.State, throughputObjective string, profiles map[string]common.Profile) ([]common.State, []float64, []planner.Action, error) { last := state + var candidates []common.State + var utilities []float64 + var actions []planner.Action for i := len(state.CurrentPods) - 1; i < scale.cfg.MaxPods; i++ { + found := false newState := last.DeepCopy() newState.CurrentPods["dummy@"+strconv.Itoa(i)] = common.PodState{Availability: averageAvailability(newState.CurrentPods)} for k := range state.Intent.Objectives { @@ -105,22 +108,39 @@ func (scale ScaleOutActuator) findState(state *common.State, goal *common.State, return &ScaleOutEffect{} }) if err != nil { - return common.State{}, fmt.Errorf("no valid effect data found in knowledge base: %s - %v", err, res) + return nil, nil, nil, fmt.Errorf("no valid effect data found in knowledge base: %s - %v", err, res) } if len(newState.CurrentPods) > res.(*ScaleOutEffect).ReplicaRange[1] { - return common.State{}, fmt.Errorf("scaling out further won't help - known replica Range: %v", res.(*ScaleOutEffect).ReplicaRange) + return nil, nil, nil, fmt.Errorf("scaling out further won't help - known replica Range: %v", res.(*ScaleOutEffect).ReplicaRange) } newState.Intent.Objectives[k] = predictLatency(res.(*ScaleOutEffect).Popt, (state.Intent.Objectives[throughputObjective]*res.(*ScaleOutEffect).ThroughputScale[0])+res.(*ScaleOutEffect).ThroughputScale[1], len(newState.CurrentPods)) + if newState.Intent.Objectives[k] <= goal.Intent.Objectives[k] { + found = true + } } else if profiles[k].ProfileType == common.ProfileTypeFromText("availability") { newState.Intent.Objectives[k] = controller.PodSetAvailability(newState.CurrentPods) + if newState.Intent.Objectives[k] >= goal.Intent.Objectives[k] { + found = true + } } } + // if at least one objective is improved by scaling mark it as a candidate; or... + if found { + candidates = append(candidates, newState) + utilities = append(utilities, 0.9+(float64(len(newState.CurrentPods))/float64(scale.cfg.MaxPods))*(1.0/goal.Intent.Priority)) + actions = append(actions, planner.Action{ + Name: scale.Name(), Properties: map[string]int64{"factor": int64(len(newState.CurrentPods) - len(state.CurrentPods))}}) + } + // ... when all are satisfied we can stop. if newState.IsBetter(goal, profiles) { - return newState, nil + break } last = &newState } - return common.State{}, fmt.Errorf("could not find a follow-up state for: %v", state) + if len(candidates) == 0 { + return nil, nil, nil, fmt.Errorf("could not find a follow-up state for: %v", state) + } + return candidates, utilities, actions, nil } func (scale ScaleOutActuator) NextState(state *common.State, goal *common.State, profiles map[string]common.Profile) ([]common.State, []float64, []planner.Action) { @@ -139,7 +159,7 @@ func (scale ScaleOutActuator) NextState(state *common.State, goal *common.State, return nil, nil, nil } - newState, err := scale.findState(state, goal, throughputObjective, profiles) + states, utils, actions, err := scale.findStates(state, goal, throughputObjective, profiles) if err != nil { klog.Warningf("Could not determine scale out factor: %s", err) if _, ok := state.CurrentPods["proactiveTemp"]; !ok && len(state.CurrentPods) < scale.cfg.MaxProActiveScaleOut { @@ -155,10 +175,7 @@ func (scale ScaleOutActuator) NextState(state *common.State, goal *common.State, } return nil, nil, nil } - utility := 0.9 + (float64(len(newState.CurrentPods))/float64(scale.cfg.MaxPods))*(1.0/goal.Intent.Priority) - return []common.State{newState}, []float64{utility}, []planner.Action{ - {Name: scale.Name(), Properties: map[string]int64{"factor": int64(len(newState.CurrentPods) - len(state.CurrentPods))}}, - } + return states, utils, actions } func (scale ScaleOutActuator) Perform(state *common.State, plan []planner.Action) { @@ -189,7 +206,7 @@ func (scale ScaleOutActuator) Perform(state *common.State, plan []planner.Action return err } // conversion to int32 is ok - as we have a MaxPods defined - res.Spec.Replicas = getInt32Pointer(*res.Spec.Replicas + int32(factor)) //nolint:gosec // explanation: casting len to int64 for API compatibility + res.Spec.Replicas = getInt32Pointer(*res.Spec.Replicas + int32(factor)) // #nosec G115 if *res.Spec.Replicas > 0 { _, updateErr := scale.apps.AppsV1().Deployments(namespace).Update(context.TODO(), res, metaV1.UpdateOptions{}) return updateErr @@ -207,7 +224,7 @@ func (scale ScaleOutActuator) Perform(state *common.State, plan []planner.Action return err } // conversion to int32 is ok - as we have a MaxPods defined - res.Spec.Replicas = getInt32Pointer(*res.Spec.Replicas + int32(factor)) //nolint:gosec // explanation: casting len to int64 for API compatibility + res.Spec.Replicas = getInt32Pointer(*res.Spec.Replicas + int32(factor)) // #nosec G115 if *res.Spec.Replicas > 0 { _, updateErr := scale.apps.AppsV1().ReplicaSets(namespace).Update(context.TODO(), res, metaV1.UpdateOptions{}) return updateErr diff --git a/pkg/planner/actuators/scaling/scale_out_test.go b/pkg/planner/actuators/scaling/scale_out_test.go index a1778b0..204861f 100644 --- a/pkg/planner/actuators/scaling/scale_out_test.go +++ b/pkg/planner/actuators/scaling/scale_out_test.go @@ -64,16 +64,10 @@ func (f *scaleOutActuatorFixture) newScaleOutTestActuator() *ScaleOutActuator { func TestScaleNextStateForSuccess(t *testing.T) { f := newScaleOutActuatorFixture(t) actuator := f.newScaleOutTestActuator() - state := common.State{Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0}}} + state := common.State{Intent: common.Intent{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0}}} goal := common.State{} goal.Intent.Objectives = map[string]float64{"p99": 10.0} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} actuator.NextState(&state, &goal, profiles) } @@ -103,14 +97,8 @@ func TestScalePerformForSuccess(t *testing.T) { // TestScaleEffectForSuccess tests for success. func TestScaleEffectForSuccess(t *testing.T) { f := newScaleOutActuatorFixture(t) - state := common.State{Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0, "throughput": 10}}} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}, "throughput": {ProfileType: common.ProfileTypeFromText("throughput")}} + state := common.State{Intent: common.Intent{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0, "throughput": 10}}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, "throughput": {ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false}} actuator := f.newScaleOutTestActuator() actuator.Effect(&state, profiles) } @@ -123,13 +111,7 @@ func TestScaleNextStateForFailure(t *testing.T) { actuator := f.newScaleOutTestActuator() state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -143,7 +125,7 @@ func TestScaleNextStateForFailure(t *testing.T) { goal := common.State{} goal.Intent.Objectives = map[string]float64{"default/p99": 3.0, "default/rps": 0.0, "default/availability": 0.999} profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, } // no throughput is being tracked. @@ -201,14 +183,8 @@ func TestScalePerformForFailure(t *testing.T) { func TestScaleEffectForFailure(t *testing.T) { f := newScaleOutActuatorFixture(t) actuator := f.newScaleOutTestActuator() - state := common.State{Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0, "throughput": 10}}} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}, "throughput": {ProfileType: common.ProfileTypeFromText("throughput")}} + state := common.State{Intent: common.Intent{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0, "throughput": 10}}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, "throughput": {ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false}} actuator.cfg.Script = "abc.xyz" actuator.Effect(&state, profiles) @@ -222,13 +198,7 @@ func TestScaleNextStateForSanity(t *testing.T) { actuator := f.newScaleOutTestActuator() state := common.State{ - Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{ + Intent: common.Intent{ Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", @@ -243,21 +213,29 @@ func TestScaleNextStateForSanity(t *testing.T) { goal.Intent.Priority = 1.0 goal.Intent.Objectives = map[string]float64{"default/p99": 3.0, "default/rps": 0.0, "default/availability": 0.999} profiles := map[string]common.Profile{ - "default/p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "default/rps": {ProfileType: common.ProfileTypeFromText("throughput")}, - "default/availability": {ProfileType: common.ProfileTypeFromText("availability")}, + "default/p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "default/rps": {ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false}, + "default/availability": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, } states, utilities, actions := actuator.NextState(&state, &goal, profiles) - if len(states) < 1 || len(utilities) < 1 || len(actions) < 1 { - t.Errorf("Resultsets are empty: %v, %v, %v.", states, utilities, actions) + if len(states) != 2 || len(utilities) != 2 || len(actions) != 2 { + t.Errorf("Resultsets do not exactly contain 2 entries: %v, %v, %v.", states, utilities, actions) } - // check if results match for scale-out - if len(states[0].CurrentPods) != 3 || actions[0].Name != actuator.Name() || actions[0].Properties.(map[string]int64)["factor"] != 2 { - t.Errorf("Expected a scale out by factor of 2 - got: %v.", actions[0]) + solutions := 0 + for i, item := range actions { + if item.Properties.(map[string]int64)["factor"] == 2 { + // to achieve p99 & availability goal. + if utilities[i] < 0.95 { + solutions++ + } + } else if item.Properties.(map[string]int64)["factor"] == 1 { + // to achieve availability goal. + solutions++ + } } - if utilities[0] > 0.95 { - t.Errorf("Expected utility to be < 1.0 - got: %v.", utilities) + if solutions != 2 { + t.Errorf("Sth went wrong expected 2 actions: %v.", actions) } // empty results if no solution can be found. @@ -358,14 +336,8 @@ func TestScalePerformForSanity(t *testing.T) { func TestScaleEffectForSanity(t *testing.T) { f := newScaleOutActuatorFixture(t) // not much to do here, as this will "just" trigger a python script. - state := common.State{Intent: struct { - Key string - Priority float64 - TargetKey string - TargetKind string - Objectives map[string]float64 - }{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0}}} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}} + state := common.State{Intent: common.Intent{Key: "default/my-objective", Priority: 1.0, TargetKey: "default/my-deployment", TargetKind: "Deployment", Objectives: map[string]float64{"p99": 20.0}}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} actuator := f.newScaleOutTestActuator() // will cause a logging warning. actuator.Effect(&state, profiles) diff --git a/pkg/planner/astar/astar_planner_test.go b/pkg/planner/astar/astar_planner_test.go index bce930c..50557be 100644 --- a/pkg/planner/astar/astar_planner_test.go +++ b/pkg/planner/astar/astar_planner_test.go @@ -468,7 +468,7 @@ func TestCreatePlanForSuccess(_ *testing.T) { CurrentData: nil, } - profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} aPlanner := f.newTestPlanner(false) defer aPlanner.Stop() aPlanner.CreatePlan(start, goal, profiles) @@ -500,7 +500,7 @@ func TestTriggerEffectForSuccess(_ *testing.T) { TargetKind: "", Objectives: map[string]float64{"p99": 100}, }} - profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} aPlanner := f.newTestPlanner(false) defer aPlanner.Stop() aPlanner.TriggerEffect(state, profiles) @@ -589,7 +589,7 @@ func TestCreatePlanForSanity(t *testing.T) { CurrentPods: nil, CurrentData: nil, } - profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} testCase.planner = testCase.plannerCrt(testCase.fixture) testCase.stubs = testCase.stubsCrt(testCase.fixture) res := testCase.planner.CreatePlan(start, goal, profiles) @@ -657,7 +657,7 @@ func TestShortCutForSanity(t *testing.T) { CurrentPods: nil, CurrentData: nil, } - profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} testCase.planner = testCase.plannerCrt(testCase.fixture) testCase.stubs = testCase.stubsCrt(testCase.fixture) res := testCase.planner.CreatePlan(start, goal, profiles) @@ -701,7 +701,7 @@ func TestOpportunisticPlannerForSanity(t *testing.T) { CurrentPods: nil, CurrentData: nil, } - profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} testCase.planner = testCase.plannerCrt(testCase.fixture) testCase.stubs = testCase.stubsCrt(testCase.fixture) res := testCase.planner.CreatePlan(start, goal, profiles) @@ -735,7 +735,7 @@ func TestFaultyActuatorForSanity(t *testing.T) { start := common.State{} goal := common.State{} - profiles := map[string]common.Profile{"p72.5": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p72.5": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} res := aPlanner.CreatePlan(start, goal, profiles) if len(res) != 0 { t.Errorf("Plan should be empty.") @@ -771,7 +771,7 @@ func TestFaultyActuatorOverGrpcForSanity(t *testing.T) { start := common.State{} goal := common.State{} - profiles := map[string]common.Profile{"p72.5": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p72.5": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} res := aPlanner.CreatePlan(start, goal, profiles) if len(res) != 0 { t.Errorf("Plan should be empty.") @@ -817,7 +817,7 @@ func BenchmarkCreatePlan(b *testing.B) { CurrentPods: nil, CurrentData: nil, } - profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency")}} + profiles := map[string]common.Profile{"p99latency": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}} plnr := f.newTestPlanner(false) defer plnr.Stop() diff --git a/pkg/tests/energy_effieciency_test.go b/pkg/tests/energy_effieciency_test.go new file mode 100644 index 0000000..d90e7af --- /dev/null +++ b/pkg/tests/energy_effieciency_test.go @@ -0,0 +1,72 @@ +package tests + +import ( + "flag" + "testing" + + "github.com/intel/intent-driven-orchestration/pkg/planner/actuators" + + "github.com/intel/intent-driven-orchestration/pkg/common" + "github.com/intel/intent-driven-orchestration/pkg/planner/actuators/energy" + "github.com/intel/intent-driven-orchestration/pkg/planner/actuators/scaling" + "k8s.io/klog/v2" +) + +func init() { + var fs flag.FlagSet + klog.InitFlags(&fs) + err := fs.Set("v", "2") + if err != nil { + klog.Errorf("Ohoh unable to set log level: %v.", err) + } +} + +func TestPowerForSanity(t *testing.T) { + defaultsConfig, err1 := common.LoadConfig("traces/defaults.json", func() interface{} { + return &common.Config{} + }) + cpuScaleConfig, err2 := common.LoadConfig("traces/cpu_scale.json", func() interface{} { + return &scaling.CPUScaleConfig{} + }) + if err1 != nil || err2 != nil { + t.Errorf("Could not load config files!") + } + + cfg := energy.PowerActuatorConfig{ + Prediction: "../.././pkg/planner/actuators/energy/analytics/test_predict.py", + Analytics: "../.././pkg/planner/actuators/energy/analytics/test_analytics.py", + PowerProfiles: []string{"None", "power.intel.com/balance-power", "power.intel.com/balance-performance", "power.intel.com/performance"}, + PythonInterpreter: "python3", + RenewableLimit: 0.75, + StepDown: 2, + } + registry := map[string]actuatorSetup{ + "NewCPUScaleActuator": {scaling.NewCPUScaleActuator, *cpuScaleConfig.(*scaling.CPUScaleConfig)}, + "NewPowerActuator": {energy.NewPowerActuator, cfg}, + } + + var tests = []testEnvironment{ + {name: "power_efficiency", effectsFilename: "traces/trace_2/effects.json", eventsFilename: "traces/trace_2/events.json", defaults: defaultsConfig.(*common.Config), actuators: registry}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runTrace(tt, t, cleanUpWithPower) + }) + } +} + +// cleanUp makes sure we stop the prediction service. +func cleanUpWithPower(actuators []actuators.Actuator) { + for _, actuator := range actuators { + klog.Infof("Cleaning up actuator '%s'...", actuator.Name()) + switch v := actuator.(type) { + case *energy.PowerActuator: + err := v.Cmd.Process.Kill() + if err != nil { + klog.Errorf("Could not cleanup actuator: %v.", err) + } + default: + klog.Infof("Actuator without cleanup needs: %v.", v) + } + } +} diff --git a/pkg/tests/full_framework_test.go b/pkg/tests/full_framework_test.go index 6f0978c..2538383 100644 --- a/pkg/tests/full_framework_test.go +++ b/pkg/tests/full_framework_test.go @@ -24,6 +24,7 @@ import ( informers "github.com/intel/intent-driven-orchestration/pkg/generated/informers/externalversions" "github.com/intel/intent-driven-orchestration/pkg/planner" "github.com/intel/intent-driven-orchestration/pkg/planner/actuators" + "github.com/intel/intent-driven-orchestration/pkg/planner/actuators/platform" "github.com/intel/intent-driven-orchestration/pkg/planner/actuators/scaling" "github.com/intel/intent-driven-orchestration/pkg/planner/astar" appsV1 "k8s.io/api/apps/v1" @@ -92,11 +93,15 @@ func (t fileTracer) GetEffect(name string, group string, profileName string, _ i return tmp, nil } else if group == "vertical_scaling" { tmp := constructor().(*scaling.CPUScaleEffect) - popt := [3]float64{} - for i, v := range data["popt"].([]interface{}) { - popt[i] = v.(float64) + rawPopt := data["popt"].([]interface{}) + popt := make([][3]float64, len(rawPopt)) + for i, raw := range rawPopt { + triple := raw.([]interface{}) + for j := 0; j < 3; j++ { + popt[i][j] = triple[j].(float64) + } } - tmp.Popt = popt + tmp.Popts = popt return tmp, nil } return nil, nil @@ -143,6 +148,7 @@ type testFixture struct { k8sInformer k8sInformers.SharedInformerFactory intentInformer informers.SharedInformerFactory tracer fileTracer + actuators []actuators.Actuator prometheus prometheusDummy prometheusServer *http.Server ticker chan planEvent @@ -307,6 +313,7 @@ func (f *testFixture) newTestSetup(env testEnvironment, stopper chan struct{}) c result := res[0].Interface() actuatorList = append(actuatorList, result.(actuators.Actuator)) } + f.actuators = actuatorList plnr := astar.NewAPlanner(actuatorList, *env.defaults) defer plnr.Stop() @@ -341,13 +348,14 @@ func (f *testFixture) newTestSetup(env testEnvironment, stopper chan struct{}) c // Event represents an entry from an events' collection. type Event struct { - Intent string `json:"name"` - Current map[string]float64 `json:"current_objectives"` - Desired map[string]float64 `json:"desired_objectives"` - Pods map[string]map[string]interface{} `json:"pods"` - Resources map[string]int64 `json:"resources"` - Plan []map[string]interface{} `json:"plan"` - Data map[string]map[string]float64 `json:"data"` + Intent string `json:"name"` + Current map[string]float64 `json:"current_objectives"` + Desired map[string]float64 `json:"desired_objectives"` + Pods map[string]map[string]interface{} `json:"pods"` + Resources map[string]int64 `json:"resources"` + Annotations map[string]string `json:"annotations"` + Plan []map[string]interface{} `json:"plan"` + Data map[string]map[string]float64 `json:"data"` } // Effect represents an entry in an effects' collection. @@ -375,7 +383,7 @@ func parseTrace(filename string) []Event { tmp, err1 := io.ReadAll(trace) err2 := json.Unmarshal(tmp, &events) if err1 != nil || err2 != nil { - klog.Errorf("Could not read and/or unmarshal trace: %v-%v.", err1, err2) + klog.Errorf("Could not read and/or unmarshal trace %s: %v-%v.", filename, err1, err2) } return events } @@ -424,8 +432,10 @@ func (f *testFixture) setupProfiles(profiles map[string]float64) { for name := range profiles { tmp := strings.Split(name, "/") typeName := "throughput" + minimize := false if strings.Contains(tmp[1], "latency") { typeName = "latency" + minimize = true } profile := &v1alpha1.KPIProfile{ ObjectMeta: metaV1.ObjectMeta{ @@ -433,7 +443,8 @@ func (f *testFixture) setupProfiles(profiles map[string]float64) { Namespace: tmp[0], }, Spec: v1alpha1.KPIProfileSpec{ - KPIType: typeName, + KPIType: typeName, + Minimize: minimize, }, } _, err := f.intentClient.IdoV1alpha1().KPIProfiles(tmp[0]).Create(context.TODO(), profile, metaV1.CreateOptions{}) @@ -446,12 +457,12 @@ func (f *testFixture) setupProfiles(profiles map[string]float64) { } // setWorkloadState updates the deployment and pod specs. -func (f *testFixture) setWorkloadState(pods map[string]map[string]interface{}, resources map[string]int64) { +func (f *testFixture) setWorkloadState(pods map[string]map[string]interface{}, resources map[string]int64, annotations map[string]string) { // FIXME: current we do not store information on the workload - we could pick it up from a manifest later on. // if deployment does not exist - add it. res, err := f.k8sClient.AppsV1().Deployments("default").Get(context.TODO(), "function-deployment", metaV1.GetOptions{}) if err != nil || res == nil { - repl := int32(len(pods)) //nolint:gosec // explanation: casting len to int64 for API compatibility + repl := int32(len(pods)) //nolint:gosec // explanation: casting len to int32 for API compatibility deployment := &appsV1.Deployment{ ObjectMeta: metaV1.ObjectMeta{ Name: "function-deployment", @@ -490,9 +501,10 @@ func (f *testFixture) setWorkloadState(pods map[string]map[string]interface{}, r if err != nil || res == nil { pod := &coreV1.Pod{ ObjectMeta: metaV1.ObjectMeta{ - Name: key, - Labels: map[string]string{"app": "sample-function"}, - Namespace: "default", + Name: key, + Labels: map[string]string{"app": "sample-function"}, + Namespace: "default", + Annotations: annotations, }, Status: coreV1.PodStatus{ Phase: coreV1.PodPhase(pods[key]["state"].(string)), @@ -628,12 +640,12 @@ func (f *testFixture) setCurrentData(vals map[string]map[string]float64) { } // comparePlans compares two planes and returns false if they are not the same. -func comparePlans(onePlan []map[string]interface{}, anotherPlan []planner.Action) bool { - if len(anotherPlan) != len(onePlan) { +func comparePlans(oldPlanSet []map[string]interface{}, newActions []planner.Action) bool { + if len(newActions) != len(oldPlanSet) { return false } var oldPlan []planner.Action - for _, entry := range onePlan { + for _, entry := range oldPlanSet { tmp := planner.Action{ Name: entry["name"].(string), Properties: entry["properties"], @@ -641,18 +653,18 @@ func comparePlans(onePlan []map[string]interface{}, anotherPlan []planner.Action oldPlan = append(oldPlan, tmp) } for i, item := range oldPlan { - if item.Name != anotherPlan[i].Name { - klog.Infof("Expected action name: %v - got %v", item.Name, anotherPlan[i].Name) + if item.Name != newActions[i].Name { + klog.Infof("Expected action name: %v - got %v", item.Name, newActions[i].Name) return false } one := fmt.Sprintf("%v", item.Properties) - another := fmt.Sprintf("%v", anotherPlan[i].Properties) + another := fmt.Sprintf("%v", newActions[i].Properties) if one != another && item.Name != "rmPod" { klog.Infof("Expected property: %v - got %v", one, another) return false } else if item.Name == "rmPod" { if one != another && len(one) == len(another) { - klog.Warningf("Not super sure - but looks ok: %v - %v", item.Properties, anotherPlan[i].Properties) + klog.Warningf("Not super sure - but looks ok: %v - %v", item.Properties, newActions[i].Properties) } else if one != another { klog.Infof("This does not look right; expected: %v - got %v", one, another) return false @@ -663,7 +675,7 @@ func comparePlans(onePlan []map[string]interface{}, anotherPlan []planner.Action } // runTrace tries to retrace a single trace. -func runTrace(env testEnvironment, t *testing.T) { +func runTrace(env testEnvironment, t *testing.T, callback func([]actuators.Actuator)) { f := newTestFixture(t) stopChannel := make(chan struct{}) defer close(stopChannel) @@ -676,7 +688,7 @@ func runTrace(env testEnvironment, t *testing.T) { f.setupIntent(events[0].Intent, events[0].Desired) f.setCurrentObjectives(events[0].Current) f.setCurrentData(events[0].Data) - f.setWorkloadState(events[0].Pods, events[0].Resources) + f.setWorkloadState(events[0].Pods, events[0].Resources, events[0].Annotations) f.checkPrometheus() <-f.ticker // although we use first entry for setup, we should wait for first plan; but don't need to compare. @@ -685,7 +697,7 @@ func runTrace(env testEnvironment, t *testing.T) { fmt.Println(strconv.Itoa(i) + "----") f.setCurrentObjectives(events[i].Current) f.setCurrentData(events[i].Data) - f.setWorkloadState(events[i].Pods, events[i].Resources) + f.setWorkloadState(events[i].Pods, events[i].Resources, events[i].Annotations) f.setDesiredObjectives(events[i].Intent, events[i].Desired) f.tracer.stepIndex(i) @@ -695,6 +707,12 @@ func runTrace(env testEnvironment, t *testing.T) { } } + // cleanup actuators through a quick callback. (if needed) + if callback != nil { + callback(f.actuators) + } + + // cleanup telemetry stuff err := f.prometheusServer.Shutdown(context.TODO()) if err != nil { klog.Errorf("Error while shutdown of prometheus server: %v", err) @@ -717,7 +735,10 @@ func TestTracesForSanity(t *testing.T) { cpuScaleConfig, err4 := common.LoadConfig("traces/cpu_scale.json", func() interface{} { return &scaling.CPUScaleConfig{} }) - if err1 != nil || err2 != nil || err3 != nil || err4 != nil { + rdtConfig, err5 := common.LoadConfig("traces/rdt.json", func() interface{} { + return &platform.RdtConfig{} + }) + if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil { t.Errorf("Could not load config files!") } @@ -728,12 +749,31 @@ func TestTracesForSanity(t *testing.T) { {name: "horizontal_vertical_scaling", effectsFilename: "traces/trace_1/effects.json", eventsFilename: "traces/trace_1/events.json", defaults: defaultsConfig.(*common.Config), actuators: map[string]actuatorSetup{ "NewCPUScaleActuator": {scaling.NewCPUScaleActuator, *cpuScaleConfig.(*scaling.CPUScaleConfig)}, "NewRmPodActuator": {scaling.NewRmPodActuator, *rmPodConfig.(*scaling.RmPodConfig)}, - "NewScaleOutActuator": {scaling.NewScaleOutActuator, *scaleOutConfig.(*scaling.ScaleOutConfig)}, - }}, + "NewScaleOutActuator": {scaling.NewScaleOutActuator, *scaleOutConfig.(*scaling.ScaleOutConfig)}}, + }, + {name: "rdt_trace", effectsFilename: "traces/trace_rdt/effects.json", eventsFilename: "traces/trace_rdt/events.json", defaults: defaultsConfig.(*common.Config), actuators: map[string]actuatorSetup{ + "NewCPUScaleActuator": {scaling.NewCPUScaleActuator, *cpuScaleConfig.(*scaling.CPUScaleConfig)}, + "NewRDTActuator": {platform.NewRdtActuator, *rdtConfig.(*platform.RdtConfig)}}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - runTrace(tt, t) + runTrace(tt, t, cleanUp) }) } } + +func cleanUp(actuators []actuators.Actuator) { + for _, actuator := range actuators { + klog.Infof("Cleaning up actuator '%s'...", actuator.Name()) + switch v := actuator.(type) { + case *platform.RdtActuator: + err := v.Cmd.Process.Kill() + if err != nil { + klog.Errorf("Could not cleanup: %v", err) + } + default: + klog.Infof("Actuator without cleanup needs: %v.", v) + } + } +} diff --git a/pkg/tests/full_planner_test.go b/pkg/tests/full_planner_test.go index 6624947..b16ea17 100644 --- a/pkg/tests/full_planner_test.go +++ b/pkg/tests/full_planner_test.go @@ -60,9 +60,9 @@ func setupTestCase() (common.State, common.State, map[string]common.Profile) { }, } profiles := map[string]common.Profile{ - "p99": {ProfileType: common.ProfileTypeFromText("latency")}, - "rps": {ProfileType: common.ProfileTypeFromText("throughput")}, - "availability": {ProfileType: common.ProfileTypeFromText("availability")}, + "p99": {ProfileType: common.ProfileTypeFromText("latency"), Minimize: true}, + "rps": {ProfileType: common.ProfileTypeFromText("throughput"), Minimize: false}, + "availability": {ProfileType: common.ProfileTypeFromText("availability"), Minimize: false}, } return start, goal, profiles } diff --git a/pkg/tests/traces/cpu_scale.json b/pkg/tests/traces/cpu_scale.json index 9270904..c5921be 100644 --- a/pkg/tests/traces/cpu_scale.json +++ b/pkg/tests/traces/cpu_scale.json @@ -4,5 +4,6 @@ "cpu_max": 4000, "cpu_rounding": 100, "cpu_safeguard_factor": 0.95, + "boost_factor": 1.0, "max_proactive_cpu": 0 } diff --git a/pkg/tests/traces/defaults.json b/pkg/tests/traces/defaults.json index 508be6d..cd66ddb 100644 --- a/pkg/tests/traces/defaults.json +++ b/pkg/tests/traces/defaults.json @@ -3,7 +3,15 @@ "task_channel_length": 100, "controller_timeout": 3, "plan_cache_ttl": 1000, - "plan_cache_timeout": 100 + "plan_cache_timeout": 100, + "telemetry_endpoint": "http://127.0.0.1:39090/data", + "host_field": "host", + "metrics": [ + { + "name": "renewable_energy_ratio", + "query": "renewable_energy_ratio@%s" + } + ] }, "monitor": { "profile": { diff --git a/pkg/tests/traces/rdt.json b/pkg/tests/traces/rdt.json new file mode 100644 index 0000000..b9e99b4 --- /dev/null +++ b/pkg/tests/traces/rdt.json @@ -0,0 +1,7 @@ +{ + "interpreter": "python3", + "analytics_script": "None", + "prediction_script": "traces/trace_rdt/predict.py", + "options": ["None", "besteffort", "burstable", "guaranteed"], + "annotation_name": "rdt.resources.beta.kubernetes.io/pod" +} diff --git a/pkg/tests/traces/trace_0/effects.json b/pkg/tests/traces/trace_0/effects.json index 6a6aa42..6c143e0 100644 --- a/pkg/tests/traces/trace_0/effects.json +++ b/pkg/tests/traces/trace_0/effects.json @@ -13,9 +13,11 @@ 10 ], "popt": [ - 100, - 0.2, - 50 + [ + 100, + 0.2, + 50 + ] ], "trainingFeatures": [ "cpus" @@ -39,9 +41,11 @@ 10 ], "popt": [ - 100, - 0.2, - 50 + [ + 100, + 0.2, + 50 + ] ], "trainingFeatures": [ "cpus" @@ -65,9 +69,11 @@ 10 ], "popt": [ - 200, - 0.2, - 50 + [ + 200, + 0.2, + 50 + ] ], "trainingFeatures": [ "cpus" diff --git a/pkg/tests/traces/trace_0/events.json b/pkg/tests/traces/trace_0/events.json index ffd5897..c002252 100644 --- a/pkg/tests/traces/trace_0/events.json +++ b/pkg/tests/traces/trace_0/events.json @@ -84,7 +84,7 @@ "qosclass": "Guaranteed" } }, - "data": {}, + "data": [], "plan": [ { "name": "scaleCPU", diff --git a/pkg/tests/traces/trace_1/effects.json b/pkg/tests/traces/trace_1/effects.json index f00bb55..959ffdb 100644 --- a/pkg/tests/traces/trace_1/effects.json +++ b/pkg/tests/traces/trace_1/effects.json @@ -1,198 +1,212 @@ -[{ - "_id": { - "$oid": "65b7a9d5d8c0a5577562fb5e" - }, - "name": "default/my-intent", - "profileName": "default/p50latency", - "group": "scaling", - "data": { - "throughputRange": [ - 11.7, - 37.4 - ], - "throughputScale": [ - 0.1556420233463035, - -0.8210116731517509 - ], - "replicaRange": [ - 1, - 4 - ], - "popt": [ - 1.7953060759736734, - 0.7107664907145342, - 0.08418771034672608, - 0.16712977859691586 - ], - "trainingFeatures": [ - "default/throughput", - "replicas" - ], - "targetFeature": "default/p50latency" - }, - "timestamp": { - "$date": "2024-01-29T14:36:21.811Z" - }, - "static": true -},{ - "_id": { - "$oid": "65b7a9d22130241fce26d847" - }, - "name": "default/my-intent", - "profileName": "default/p95latency", - "group": "scaling", - "data": { - "throughputRange": [ - 6.4, - 37.4 - ], - "throughputScale": [ - 0.12903225806451613, - 0.17419354838709677 - ], - "replicaRange": [ - 1, - 4 - ], - "popt": [ - 1.876236766451076, - 0.3479978871129307, - 0.03630452074498126, - 0.05612263537403011 - ], - "trainingFeatures": [ - "default/throughput", - "replicas" - ], - "targetFeature": "default/p95latency" - }, - "timestamp": { - "$date": "2024-01-29T14:36:18.291Z" - }, - "static": true -},{ - "_id": { - "$oid": "65b7a9d8f5a8afa4802bee95" - }, - "name": "default/my-intent", - "profileName": "default/p99latency", - "group": "scaling", - "data": { - "throughputRange": [ - 6.4, - 37.2 - ], - "throughputScale": [ - 0.12987012987012986, - 0.1688311688311689 - ], - "replicaRange": [ - 1, - 4 - ], - "popt": [ - 1.7340978705359251, - 0.32188728156949387, - 0.02992459259216999, - 0.042336575288062825 - ], - "trainingFeatures": [ - "default/throughput", - "replicas" - ], - "targetFeature": "default/p99latency" - }, - "timestamp": { - "$date": "2024-01-29T14:36:24.871Z" - }, - "static": true -},{ - "_id": { - "$oid": "65b7a9d6c34376a54780332c" - }, - "name": "default/my-intent", - "profileName": "default/p50latency", - "group": "vertical_scaling", - "data": { - "latencyRange": [ - 35.043, - 149.407 - ], - "cpuRange": [ - 0.25, - 2.5 - ], - "popt": [ - 272.3754065441032, - 3.3408376505041804, - 32.917177116938596 - ], - "trainingFeatures": [ - "cpus" - ], - "targetFeature": "default/p50latency" - }, - "timestamp": { - "$date": "2024-01-29T14:36:22.920Z" - }, - "static": true -},{ - "_id": { - "$oid": "65b7a9d4b9bd0848eca85768" - }, - "name": "default/my-intent", - "profileName": "default/p95latency", - "group": "vertical_scaling", - "data": { - "latencyRange": [ - 39.815, - 245.714 - ], - "cpuRange": [ - 0.25, - 2.5 - ], - "popt": [ - 269.59131728063625, - 2.225499837045876, - 54.44974387771397 - ], - "trainingFeatures": [ - "cpus" - ], - "targetFeature": "default/p95latency"}, - "timestamp": { - "$date": "2024-01-29T14:36:20.311Z" - }, - "static": true -},{ - "_id": { - "$oid": "65b7a9d14895657c322f7e6c" - }, - "name": "default/my-intent", - "profileName": "default/p99latency", - "group": "vertical_scaling", - "data": { - "latencyRange": [ - 46.925, - 289.143 - ], - "cpuRange": [ - 0.25, - 2.5 - ], - "popt": [ - 248.69930654974348, - 1.8325502392260382, - 66.53386004054035 - ], - "trainingFeatures": [ - "cpus" - ], - "targetFeature": "default/p99latency" - }, - "timestamp": { - "$date": "2024-01-29T14:36:17.619Z" - }, - "static": true -}] +[ + { + "_id": { + "$oid": "65b7a9d5d8c0a5577562fb5e" + }, + "name": "default/my-intent", + "profileName": "default/p50latency", + "group": "scaling", + "data": { + "throughputRange": [ + 11.7, + 37.4 + ], + "throughputScale": [ + 0.1556420233463035, + -0.8210116731517509 + ], + "replicaRange": [ + 1, + 4 + ], + "popt": [ + 1.7953060759736734, + 0.7107664907145342, + 0.08418771034672608, + 0.16712977859691586 + ], + "trainingFeatures": [ + "default/throughput", + "replicas" + ], + "targetFeature": "default/p50latency" + }, + "timestamp": { + "$date": "2024-01-29T14:36:21.811Z" + }, + "static": true + }, + { + "_id": { + "$oid": "65b7a9d22130241fce26d847" + }, + "name": "default/my-intent", + "profileName": "default/p95latency", + "group": "scaling", + "data": { + "throughputRange": [ + 6.4, + 37.4 + ], + "throughputScale": [ + 0.12903225806451613, + 0.17419354838709677 + ], + "replicaRange": [ + 1, + 4 + ], + "popt": [ + 1.876236766451076, + 0.3479978871129307, + 0.03630452074498126, + 0.05612263537403011 + ], + "trainingFeatures": [ + "default/throughput", + "replicas" + ], + "targetFeature": "default/p95latency" + }, + "timestamp": { + "$date": "2024-01-29T14:36:18.291Z" + }, + "static": true + }, + { + "_id": { + "$oid": "65b7a9d8f5a8afa4802bee95" + }, + "name": "default/my-intent", + "profileName": "default/p99latency", + "group": "scaling", + "data": { + "throughputRange": [ + 6.4, + 37.2 + ], + "throughputScale": [ + 0.12987012987012986, + 0.1688311688311689 + ], + "replicaRange": [ + 1, + 4 + ], + "popt": [ + 1.7340978705359251, + 0.32188728156949387, + 0.02992459259216999, + 0.042336575288062825 + ], + "trainingFeatures": [ + "default/throughput", + "replicas" + ], + "targetFeature": "default/p99latency" + }, + "timestamp": { + "$date": "2024-01-29T14:36:24.871Z" + }, + "static": true + }, + { + "_id": { + "$oid": "65b7a9d6c34376a54780332c" + }, + "name": "default/my-intent", + "profileName": "default/p50latency", + "group": "vertical_scaling", + "data": { + "latencyRange": [ + 35.043, + 149.407 + ], + "cpuRange": [ + 0.25, + 2.5 + ], + "popt": [ + [ + 272.3754065441032, + 3.3408376505041804, + 32.917177116938596 + ] + ], + "trainingFeatures": [ + "cpus" + ], + "targetFeature": "default/p50latency" + }, + "timestamp": { + "$date": "2024-01-29T14:36:22.920Z" + }, + "static": true + }, + { + "_id": { + "$oid": "65b7a9d4b9bd0848eca85768" + }, + "name": "default/my-intent", + "profileName": "default/p95latency", + "group": "vertical_scaling", + "data": { + "latencyRange": [ + 39.815, + 245.714 + ], + "cpuRange": [ + 0.25, + 2.5 + ], + "popt": [ + [ + 269.59131728063625, + 2.225499837045876, + 54.44974387771397 + ] + ], + "trainingFeatures": [ + "cpus" + ], + "targetFeature": "default/p95latency" + }, + "timestamp": { + "$date": "2024-01-29T14:36:20.311Z" + }, + "static": true + }, + { + "_id": { + "$oid": "65b7a9d14895657c322f7e6c" + }, + "name": "default/my-intent", + "profileName": "default/p99latency", + "group": "vertical_scaling", + "data": { + "latencyRange": [ + 46.925, + 289.143 + ], + "cpuRange": [ + 0.25, + 2.5 + ], + "popt": [ + [ + 248.69930654974348, + 1.8325502392260382, + 66.53386004054035 + ] + ], + "trainingFeatures": [ + "cpus" + ], + "targetFeature": "default/p99latency" + }, + "timestamp": { + "$date": "2024-01-29T14:36:17.619Z" + }, + "static": true + } +] diff --git a/pkg/tests/traces/trace_2/effects.json b/pkg/tests/traces/trace_2/effects.json new file mode 100644 index 0000000..1b2fd50 --- /dev/null +++ b/pkg/tests/traces/trace_2/effects.json @@ -0,0 +1,30 @@ +[ + { + "name": "default/my-app-intent", + "profileName": "default/p99latency", + "group": "vertical_scaling", + "data": { + "latencyRange": [ + 50, + 150 + ], + "cpuRange": [ + 1, + 10 + ], + "popt": [ + [ + 100, + 0.2, + 50 + ] + ], + "trainingFeatures": [ + "cpus" + ], + "targetFeature": "default/p99latency" + }, + "static": true, + "timestamp": "0" + } +] \ No newline at end of file diff --git a/pkg/tests/traces/trace_2/events.json b/pkg/tests/traces/trace_2/events.json new file mode 100644 index 0000000..3aa795a --- /dev/null +++ b/pkg/tests/traces/trace_2/events.json @@ -0,0 +1,229 @@ +[ + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": -1.0 + }, + "desired_objectives": { + "default/p99latency": 150.0 + }, + "resources": {}, + "annotations": {}, + "pods": { + "pod_0": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "Burstable" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 1.0 + } + }, + "plan": null + }, + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": 200.0 + }, + "desired_objectives": { + "default/p99latency": 100.0 + }, + "resources": {}, + "annotations": {}, + "pods": { + "pod_0": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "BestEffort" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 1.0 + } + }, + "plan": [ + { + "name": "scaleCPU", + "properties": { + "value": 4000 + } + } + ] + }, + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": 200.0 + }, + "desired_objectives": { + "default/p99latency": 100.0 + }, + "resources": { + "0_cpu_requests": 4000, + "0_cpu_limits": 4000 + }, + "annotations": {}, + "pods": { + "pod_1": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 1.0 + } + }, + "plan": [ + { + "name": "setPowerProfile", + "properties": { + "profile": "power.intel.com/balance-performance" + } + } + ] + }, + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": 100.0 + }, + "desired_objectives": { + "default/p99latency": 100.0 + }, + "resources": { + "0_cpu_requests": 4000, + "0_cpu_limits": 4000, + "0_power.intel.com/balance-performance_requests": 4000 + }, + "annotations": {}, + "pods": { + "pod_2": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 1.0 + } + }, + "plan": [] + }, + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": 100.0 + }, + "desired_objectives": { + "default/p99latency": 100.0 + }, + "resources": { + "0_cpu_requests": 4000, + "0_cpu_limits": 4000, + "0_power.intel.com/balance-performance_requests": 4000 + }, + "annotations": {}, + "pods": { + "pod_2": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 0.1 + } + }, + "plan": [ + { + "name": "setPowerProfile", + "properties": { + "profile": "None" + } + } + ] + }, + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": 200.0 + }, + "desired_objectives": { + "default/p99latency": 100.0 + }, + "resources": { + "0_cpu_requests": 4000, + "0_cpu_limits": 4000 + }, + "annotations": {}, + "pods": { + "pod_3": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 0.1 + } + }, + "plan": [] + }, + { + "name": "default/my-app-intent", + "timestamp": "2023-06-01 12:00:00.000000", + "current_objectives": { + "default/p99latency": 200.0 + }, + "desired_objectives": { + "default/p99latency": 175.0 + }, + "resources": { + "0_cpu_requests": 4000, + "0_cpu_limits": 4000 + }, + "annotations": {}, + "pods": { + "pod_3": { + "availability": 1.0, + "nodename": "node01", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "renewable_energy_ratio": { + "node01": 0.8 + } + }, + "plan": [ + { + "name": "setPowerProfile", + "properties": { + "profile": "power.intel.com/balance-power" + } + } + ] + } +] \ No newline at end of file diff --git a/pkg/tests/traces/trace_rdt/effects.json b/pkg/tests/traces/trace_rdt/effects.json new file mode 100644 index 0000000..82f91e2 --- /dev/null +++ b/pkg/tests/traces/trace_rdt/effects.json @@ -0,0 +1,35 @@ +[ + { + "_id": { + "$oid": "65db7946094720d66094bfc8" + }, + "name": "default/my-intent", + "profileName": "default/p95latency", + "group": "vertical_scaling", + "data": { + "latencyRange": [ + 199.605, + 1920.0 + ], + "cpuRange": [ + 0.1, + 2.0 + ], + "popt": [ + [ + 2283.06568411986, + 4.712509450970291, + 239.33831884809487 + ] + ], + "trainingFeatures": [ + "cpus" + ], + "targetFeature": "default/p95latency" + }, + "timestamp": { + "$date": "2024-02-25T18:30:46.890Z" + }, + "static": true + } +] \ No newline at end of file diff --git a/pkg/tests/traces/trace_rdt/events.json b/pkg/tests/traces/trace_rdt/events.json new file mode 100644 index 0000000..2b9742b --- /dev/null +++ b/pkg/tests/traces/trace_rdt/events.json @@ -0,0 +1,615 @@ +[{ + "_id": { + "$oid": "65e8a5078aaaefa78b84a5e0" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:16:55.850Z" + }, + "current_objectives": { + "default/p95latency": 687.5 + }, + "desired_objectives": { + "default/p95latency": 275.0 + }, + "resources": { + "1_cpu_limits": 500, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 250, + "1_memory_requests": 536870912000, + "1_memory_limits": 536870912000 + }, + "annotations": { + "config.linkerd.io/proxy-memory-limit": "256Mi", + "kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu request for container sample-function; cpu limit for container sample-function", + "linkerd.io/inject": "enabled", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "viz.linkerd.io/tap-enabled": "true" + }, + "pods": { + "function-deployment-7955c7f497-8kqzs": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Burstable" + } + }, + "data": { + "cpu_value": { + "apollo0": 17.312844681973097 + }, + "ipc_value": { + "apollo0": 0.602549914901389 + }, + "avg_l2_miss": { + "apollo0": 713486.37 + }, + "avg_l2_hit": { + "apollo0": 25845978.95 + } + }, + "plan": [ + { + "name": "scaleCPU", + "properties": { + "value": 1000 + } + } + ] +},{ + "_id": { + "$oid": "65e8a5628aaaefa78b84a5e1" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:18:26.803Z" + }, + "current_objectives": { + "default/p95latency": 297.051 + }, + "desired_objectives": { + "default/p95latency": 275.0 + }, + "resources": { + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 1000, + "1_memory_requests": 536870912000, + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000 + }, + "annotations": { + "config.linkerd.io/proxy-memory-limit": "256Mi", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "viz.linkerd.io/tap-enabled": "true", + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-cpu-request": "100m", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/inject": "enabled", + "linkerd.io/proxy-version": "stable-2.14.8" + }, + "pods": { + "function-deployment-5bfcfbd9cc-9hcdv": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "cpu_value": { + "apollo0": 30.718091965755193 + }, + "ipc_value": { + "apollo0": 0.7575615441873361 + }, + "avg_l2_miss": { + "apollo0": 1204208.02 + }, + "avg_l2_hit": { + "apollo0": 56851448.96 + } + }, + "plan": [ + { + "name": "configureRDT", + "properties": { + "option": "burstable" + } + } + ] +},{ + "_id": { + "$oid": "65e8a5bb8aaaefa78b84a5e2" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:19:55.998Z" + }, + "current_objectives": { + "default/p95latency": 276.111 + }, + "desired_objectives": { + "default/p95latency": 275.0 + }, + "resources": { + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_memory_limits": 268435456000, + "0_cpu_limits": 100, + "1_cpu_requests": 1000, + "1_memory_requests": 536870912000, + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000 + }, + "annotations": { + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/inject": "enabled", + "rdt.resources.beta.kubernetes.io/pod": "burstable", + "viz.linkerd.io/tap-enabled": "true", + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f" + }, + "pods": { + "function-deployment-7c746868cb-4vd66": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "cpu_value": { + "apollo0": 28.50192064047692 + }, + "ipc_value": { + "apollo0": 0.8165065257446075 + }, + "avg_l2_miss": { + "apollo0": 961796.5900000001 + }, + "avg_l2_hit": { + "apollo0": 56888561.18 + } + }, + "plan": null +},{ + "_id": { + "$oid": "65e8a5e88aaaefa78b84a5e3" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:20:40.966Z" + }, + "current_objectives": { + "default/p95latency": 282.083 + }, + "desired_objectives": { + "default/p95latency": 275.0 + }, + "resources": { + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 1000, + "1_memory_requests": 536870912000, + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000 + }, + "annotations": { + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/inject": "enabled", + "rdt.resources.beta.kubernetes.io/pod": "burstable", + "viz.linkerd.io/tap-enabled": "true" + }, + "pods": { + "function-deployment-7c746868cb-4vd66": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "cpu_value": { + "apollo0": 28.61255638960727 + }, + "ipc_value": { + "apollo0": 0.8153794736611958 + }, + "avg_l2_miss": { + "apollo0": 640840.04 + }, + "avg_l2_hit": { + "apollo0": 37935522.41 + } + }, + "plan": null +},{ + "_id": { + "$oid": "65e8a6158aaaefa78b84a5e4" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:21:25.982Z" + }, + "current_objectives": { + "default/p95latency": 278.5 + }, + "desired_objectives": { + "default/p95latency": 275.0 + }, + "resources": { + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 1000, + "1_memory_requests": 536870912000, + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000 + }, + "annotations": { + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/inject": "enabled", + "rdt.resources.beta.kubernetes.io/pod": "burstable", + "viz.linkerd.io/tap-enabled": "true", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "config.linkerd.io/proxy-cpu-limit": "100m" + }, + "pods": { + "function-deployment-7c746868cb-4vd66": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "cpu_value": { + "apollo0": 24.525260486302443 + }, + "ipc_value": { + "apollo0": 0.8183643717288037 + }, + "avg_l2_miss": { + "apollo0": 963584.22 + }, + "avg_l2_hit": { + "apollo0": 56888748.96 + } + }, + "plan": null +},{ + "_id": { + "$oid": "65e8a6428aaaefa78b84a5e5" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:22:10.917Z" + }, + "current_objectives": { + "default/p95latency": 264.167 + }, + "desired_objectives": { + "default/p95latency": 275.0 + }, + "resources": { + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 1000, + "1_memory_requests": 536870912000 + }, + "annotations": { + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/inject": "enabled", + "rdt.resources.beta.kubernetes.io/pod": "burstable", + "viz.linkerd.io/tap-enabled": "true" + }, + "pods": { + "function-deployment-7c746868cb-4vd66": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "avg_l2_miss": { + "apollo0": 636668.4299999999 + }, + "avg_l2_hit": { + "apollo0": 37928655.84 + }, + "cpu_value": { + "apollo0": 24.451023845099982 + }, + "ipc_value": { + "apollo0": 0.825575225780129 + } + }, + "plan": null +},{ + "_id": { + "$oid": "65e8a6518aaaefa78b84a5e6" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:22:25.897Z" + }, + "current_objectives": { + "default/p95latency": 273.125 + }, + "desired_objectives": { + "default/p95latency": 300.0 + }, + "resources": { + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 1000, + "1_memory_requests": 536870912000, + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000 + }, + "annotations": { + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/inject": "enabled", + "rdt.resources.beta.kubernetes.io/pod": "burstable", + "viz.linkerd.io/tap-enabled": "true" + }, + "pods": { + "function-deployment-7c746868cb-4vd66": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "ipc_value": { + "apollo0": 0.8251488443366977 + }, + "avg_l2_miss": { + "apollo0": 949524.6699999999 + }, + "avg_l2_hit": { + "apollo0": 56867328.300000004 + }, + "cpu_value": { + "apollo0": 24.95138126921185 + } + }, + "plan": [ + { + "name": "configureRDT", + "properties": { + "option": "None" + } + } + ] +},{ + "_id": { + "$oid": "65e8a69c8aaaefa78b84a5e7" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:23:40.913Z" + }, + "current_objectives": { + "default/p95latency": 294.625 + }, + "desired_objectives": { + "default/p95latency": 300.0 + }, + "resources": { + "1_cpu_requests": 1000, + "1_cpu_limits": 1000, + "1_memory_limits": 536870912000, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_memory_requests": 536870912000 + }, + "annotations": { + "linkerd.io/inject": "enabled", + "linkerd.io/proxy-version": "stable-2.14.8", + "viz.linkerd.io/tap-enabled": "true", + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi" + }, + "pods": { + "function-deployment-5bfcfbd9cc-8x46n": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "cpu_value": { + "apollo0": 29.697044392243136 + }, + "ipc_value": { + "apollo0": 0.7383036137062259 + }, + "avg_l2_miss": { + "apollo0": 810030.56 + }, + "avg_l2_hit": { + "apollo0": 37762463.51 + } + }, + "plan": [ + { + "name": "scaleCPU", + "properties": { + "value": 900 + } + } + ] +},{ + "_id": { + "$oid": "65e8a6f68aaaefa78b84a5e8" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:25:10.909Z" + }, + "current_objectives": { + "default/p95latency": 297.105 + }, + "desired_objectives": { + "default/p95latency": 300.0 + }, + "resources": { + "0_memory_limits": 268435456000, + "1_cpu_requests": 900, + "1_memory_requests": 536870912000, + "1_cpu_limits": 900, + "1_memory_limits": 536870912000, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100 + }, + "annotations": { + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/proxy-version": "stable-2.14.8", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "viz.linkerd.io/tap-enabled": "true", + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/inject": "enabled" + }, + "pods": { + "function-deployment-5db7cdb8d9-vwc7f": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "ipc_value": { + "apollo0": 0.7374209286956376 + }, + "avg_l2_miss": { + "apollo0": 785523.37 + }, + "avg_l2_hit": { + "apollo0": 36876681.15 + }, + "cpu_value": { + "apollo0": 26.320305116381096 + } + }, + "plan": null +},{ + "_id": { + "$oid": "65e8a7238aaaefa78b84a5e9" + }, + "name": "default/my-intent", + "timestamp": { + "$date": "2024-03-06T17:25:55.815Z" + }, + "current_objectives": { + "default/p95latency": 297.051 + }, + "desired_objectives": { + "default/p95latency": 300.0 + }, + "resources": { + "1_memory_requests": 536870912000, + "1_cpu_limits": 900, + "1_memory_limits": 536870912000, + "0_cpu_requests": 100, + "0_memory_requests": 268435456000, + "0_cpu_limits": 100, + "0_memory_limits": 268435456000, + "1_cpu_requests": 900 + }, + "annotations": { + "config.linkerd.io/proxy-cpu-limit": "100m", + "config.linkerd.io/proxy-memory-request": "256Mi", + "linkerd.io/inject": "enabled", + "linkerd.io/trust-root-sha256": "1128a81edac74e8510e0cef0082a94e9ddeb9a1436419a84745b7431e8f4852f", + "viz.linkerd.io/tap-enabled": "true", + "config.linkerd.io/proxy-cpu-request": "100m", + "config.linkerd.io/proxy-memory-limit": "256Mi", + "linkerd.io/created-by": "linkerd/proxy-injector stable-2.14.8", + "linkerd.io/proxy-version": "stable-2.14.8" + }, + "pods": { + "function-deployment-5db7cdb8d9-vwc7f": { + "availability": 1.0, + "nodename": "apollo0", + "state": "Running", + "qosclass": "Guaranteed" + } + }, + "data": { + "cpu_value": { + "apollo0": 26.42434360361244 + }, + "ipc_value": { + "apollo0": 0.7504397981213521 + }, + "avg_l2_miss": { + "apollo0": 1175778.8600000003 + }, + "avg_l2_hit": { + "apollo0": 56420046.79 + } + }, + "plan": null +}] \ No newline at end of file diff --git a/pkg/tests/traces/trace_rdt/p95_rdt.csv b/pkg/tests/traces/trace_rdt/p95_rdt.csv new file mode 100644 index 0000000..09d8c51 --- /dev/null +++ b/pkg/tests/traces/trace_rdt/p95_rdt.csv @@ -0,0 +1,94 @@ +,default/p95latency,cpu,rdt_config,replicas,anomaly +2024-02-25T17:30:43.605Z,269.286,1.0,3,1,1 +2024-02-25T17:30:16.983Z,257.0,1.0,3,1,1 +2024-02-25T17:29:58.902Z,264.167,1.0,3,1,1 +2024-02-25T17:29:13.829Z,246.25,1.0,3,1,1 +2024-02-25T17:28:28.616Z,228.333,1.0,3,1,1 +2024-02-25T17:27:43.903Z,264.167,1.0,3,1,1 +2024-02-25T17:26:58.922Z,264.167,1.0,3,1,1 +2024-02-25T17:26:40.589Z,246.25,1.0,3,1,1 +2024-02-25T17:26:13.865Z,246.25,1.0,3,1,1 +2024-02-25T17:25:56.177Z,257.0,1.0,3,1,1 +2024-02-25T17:25:29.397Z,287.647,1.0,2,2,1 +2024-02-25T17:25:04.220Z,280.455,1.0,2,1,1 +2024-02-25T17:24:43.950Z,276.111,1.0,2,1,1 +2024-02-25T17:23:58.800Z,264.167,1.0,2,1,1 +2024-02-25T17:23:13.717Z,264.167,1.0,2,1,1 +2024-02-25T17:22:28.619Z,269.286,1.0,2,1,1 +2024-02-25T17:21:43.867Z,246.25,1.0,2,1,1 +2024-02-25T17:21:13.990Z,264.167,1.0,2,1,1 +2024-02-25T17:20:58.799Z,264.167,1.0,2,1,1 +2024-02-25T17:20:13.699Z,280.455,1.0,2,1,1 +2024-02-25T17:19:50.706Z,282.083,1.0,2,1,1 +2024-02-25T17:19:43.073Z,265.833,1.0,1,2,1 +2024-02-25T17:19:29.279Z,290.652,1.0,1,2,1 +2024-02-25T17:18:43.742Z,290.227,1.0,1,1,1 +2024-02-25T17:17:58.658Z,289.25,1.0,1,1,1 +2024-02-25T17:17:13.737Z,289.25,1.0,1,1,1 +2024-02-25T17:16:28.864Z,286.563,1.0,1,1,1 +2024-02-25T17:15:57.231Z,288.056,1.0,1,1,1 +2024-02-25T17:15:43.535Z,291.731,1.0,1,1,1 +2024-02-25T17:14:58.811Z,291.042,1.0,1,1,1 +2024-02-25T17:14:22.897Z,287.353,1.0,1,1,1 +2024-02-25T17:14:13.552Z,292.586,1.0,1,1,1 +2024-02-25T17:13:44.154Z,291.4,1.0,1,1,1 +2024-02-25T17:13:29.039Z,296.795,1.0,0,2,1 +2024-02-25T17:11:58.540Z,294.625,1.0,0,1,1 +2024-02-25T17:11:13.609Z,294.625,1.0,0,1,1 +2024-02-25T17:10:37.264Z,297.051,1.0,0,1,1 +2024-02-25T17:10:28.530Z,294.625,1.0,0,1,1 +2024-02-25T17:09:43.597Z,294.625,1.0,0,1,1 +2024-02-25T17:09:05.011Z,297.051,1.0,0,1,1 +2024-02-25T17:08:58.648Z,294.625,1.0,0,1,1 +2024-02-25T17:08:13.470Z,294.625,1.0,0,1,1 +2024-02-25T17:07:36.556Z,299.73,1.0,0,1,1 +2024-02-25T17:07:28.589Z,297.237,1.0,0,2,1 +2024-02-25T17:07:28.464Z,297.237,1.0,0,2,1 +2024-02-25T17:03:43.426Z,257.0,2.0,3,1,1 +2024-02-25T17:02:13.420Z,257.0,2.0,3,1,1 +2024-02-25T16:58:41.229Z,246.25,1.75,3,1,1 +2024-02-25T16:58:28.480Z,257.0,1.75,3,1,1 +2024-02-25T16:56:58.499Z,257.0,1.75,3,1,1 +2024-02-25T16:56:13.496Z,264.167,1.75,3,1,1 +2024-02-25T16:55:28.486Z,264.167,1.75,3,1,1 +2024-02-25T16:54:44.326Z,269.286,1.75,3,1,1 +2024-02-25T16:54:43.424Z,269.286,1.75,3,1,1 +2024-02-25T16:54:00.806Z,264.167,1.75,3,1,1 +2024-02-25T16:53:58.538Z,257.0,1.75,3,2,1 +2024-02-25T16:51:43.510Z,246.25,1.5,3,1,1 +2024-02-25T16:50:58.445Z,246.25,1.5,3,1,1 +2024-02-25T16:50:13.489Z,257.0,1.5,3,1,1 +2024-02-25T16:49:28.481Z,257.0,1.5,3,1,1 +2024-02-25T16:49:27.303Z,257.0,1.5,3,1,1 +2024-02-25T16:48:43.425Z,257.0,1.5,3,1,1 +2024-02-25T16:48:04.658Z,264.167,1.5,3,1,1 +2024-02-25T16:47:58.414Z,246.25,1.5,3,1,1 +2024-02-25T16:47:17.922Z,269.286,1.5,3,1,1 +2024-02-25T16:45:43.443Z,257.0,1.25,3,1,1 +2024-02-25T16:44:58.532Z,257.0,1.25,3,1,1 +2024-02-25T16:44:13.483Z,264.167,1.25,3,1,1 +2024-02-25T16:44:10.139Z,264.167,1.25,3,1,1 +2024-02-25T16:43:28.504Z,246.25,1.25,3,1,1 +2024-02-25T16:42:43.450Z,257.0,1.25,3,1,1 +2024-02-25T16:41:13.436Z,256.0,1.25,3,1,1 +2024-02-25T16:40:34.199Z,280.455,1.25,3,1,1 +2024-02-25T16:39:43.412Z,257.0,1.0,3,1,1 +2024-02-25T16:38:58.522Z,246.25,1.0,3,1,1 +2024-02-25T16:38:54.997Z,228.333,1.0,3,1,1 +2024-02-25T16:38:13.521Z,246.25,1.0,3,1,1 +2024-02-25T16:37:28.471Z,264.167,1.0,3,1,1 +2024-02-25T16:37:21.298Z,269.286,1.0,3,1,1 +2024-02-25T16:36:43.501Z,257.0,1.0,3,1,1 +2024-02-25T16:35:58.453Z,246.25,1.0,3,1,1 +2024-02-25T16:35:13.496Z,228.333,1.0,3,1,1 +2024-02-25T16:34:28.430Z,273.125,1.0,3,1,1 +2024-02-25T16:34:20.605Z,273.125,1.0,3,1,1 +2024-02-25T16:33:33.794Z,294.605,0.75,3,1,1 +2024-02-25T16:32:58.408Z,294.605,0.75,3,1,1 +2024-02-25T16:32:13.453Z,294.605,0.75,3,1,1 +2024-02-25T16:31:28.424Z,294.605,0.75,3,1,1 +2024-02-25T16:30:43.582Z,294.605,0.75,3,1,1 +2024-02-25T16:29:58.528Z,294.605,0.75,3,1,1 +2024-02-25T16:28:28.420Z,294.605,0.75,3,1,1 +2024-02-25T16:28:13.222Z,294.605,0.75,3,1,1 +2024-02-25T16:28:07.121Z,294.605,0.75,3,1,1 diff --git a/pkg/tests/traces/trace_rdt/predict.py b/pkg/tests/traces/trace_rdt/predict.py new file mode 100644 index 0000000..db93a9e --- /dev/null +++ b/pkg/tests/traces/trace_rdt/predict.py @@ -0,0 +1,54 @@ +import json +from wsgiref.simple_server import make_server + +import sys + +import pandas as pd +from sklearn import ensemble + +FILENAME = "traces/trace_rdt/p95_rdt.csv" +MODEL = ensemble.ExtraTreesRegressor(n_estimators=50) +FEAT_MAP = ['None', 'besteffort', 'burstable', 'guaranteed'] + + +def _train(): + df = pd.read_csv(FILENAME, index_col=0, parse_dates=True) + MODEL.fit(df[['cpu', 'rdt_config', 'replicas']], df['default/p95latency']) + + +def predict_app(environ, start_response): + """ + Predicts the effect on a latency target, or return -1.0. + """ + try: + body_size = int(environ.get('CONTENT_LENGTH', 0)) + except ValueError: + body_size = 0 + + request_body = environ['wsgi.input'].read(body_size) + body = json.loads(request_body) + + option = body['option'] + cpu = body['cpu'] + replicas = body['replicas'] + + tmp = MODEL.predict([[cpu, FEAT_MAP.index(option), replicas]]) + res = tmp[0] + + status = '200 OK' + headers = [('Content-type', 'application/json')] + start_response(status, headers) + + tmp = json.dumps({'val': res}) + return [tmp.encode()] + + +def serve(): + _train() + with make_server('127.0.0.1', 8000, predict_app) as httpd: + httpd.serve_forever() + + +if __name__ == '__main__': + sys.stdout.write(str(serve())) + sys.stdout.flush() diff --git a/plugins/cpu_scale/Dockerfile b/plugins/cpu_scale/Dockerfile index 48c089e..9a3d2d1 100644 --- a/plugins/cpu_scale/Dockerfile +++ b/plugins/cpu_scale/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.24.2 AS builder +FROM golang:1.24.6 AS builder WORKDIR /plugins @@ -11,12 +11,12 @@ RUN make prepare-build build-plugins \ && go run github.com/google/go-licenses@v1.6.0 save "./..." --save_path licenses \ && hack/additional-licenses.sh -FROM alpine:3.20 +FROM alpine:3.22 RUN adduser -D nonroot -RUN apk add --upgrade --no-cache openssl=~3.3 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.7 \ - py3-pip=~24.0 py3-scikit-learn=~1.3 -RUN pip install --break-system-packages --no-cache-dir pymongo~=4.6 +RUN apk add --upgrade --no-cache openssl=~3.5 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.9 \ + py3-pip=~25.1 py3-scikit-learn=~1.5 +RUN pip install --break-system-packages --no-cache-dir pymongo~=4.12 WORKDIR /plugins diff --git a/plugins/cpu_scale/cmd/cpu_scale.go b/plugins/cpu_scale/cmd/cpu_scale.go index cbf156b..2986970 100644 --- a/plugins/cpu_scale/cmd/cpu_scale.go +++ b/plugins/cpu_scale/cmd/cpu_scale.go @@ -52,7 +52,7 @@ func main() { klog.Fatalf("Error on generic configuration for actuator: %s", err) } err = isValidConf(cfg.PythonInterpreter, cfg.Script, cfg.CPUMax, cfg.CPURounding, cfg.MaxProActiveCPU, - cfg.CPUSafeGuardFactor, cfg.ProActiveLatencyPercentage, cfg.LookBack) + cfg.BoostFactor, cfg.CPUSafeGuardFactor, cfg.ProActiveLatencyPercentage, cfg.LookBack) if err != nil { klog.Fatalf("Error on configuration for actuator: %s", err) } @@ -75,7 +75,7 @@ func main() { } func isValidConf(interpreter, script string, confCPUMax, confCPURounding, confMaxProActiveCPU int64, - confCPUSafeGuardFactor, configProActiveLatencyPercentage float64, lookBack int) error { + boostFactor, confCPUSafeGuardFactor, configProActiveLatencyPercentage float64, lookBack int) error { if !pluginsHelper.IsStrConfigValid(interpreter) { return fmt.Errorf("invalid path to python interpreter: %s", interpreter) } @@ -99,6 +99,10 @@ func isValidConf(interpreter, script string, confCPUMax, confCPURounding, confMa return fmt.Errorf("invalid max proactive value: %d", confMaxProActiveCPU) } + if boostFactor < 0.0 || boostFactor > 10.0 { + return fmt.Errorf("invalid boost factor - needs to be in range of 0-10; was: %f", boostFactor) + } + if confCPUSafeGuardFactor <= 0 || confCPUSafeGuardFactor > 1 { return fmt.Errorf("invalid safeguard factor: %f", confCPUSafeGuardFactor) } diff --git a/plugins/cpu_scale/cmd/cpu_scale_test.go b/plugins/cpu_scale/cmd/cpu_scale_test.go index ff44b76..9731360 100644 --- a/plugins/cpu_scale/cmd/cpu_scale_test.go +++ b/plugins/cpu_scale/cmd/cpu_scale_test.go @@ -14,6 +14,7 @@ func TestIsValidConf(t *testing.T) { cpuMax int64 cpuRounding int64 maxProActiveCPU int64 + boostFactor float64 cpuSafeGuardFactor float64 proActiveLatencyPercentage float64 lookBack int @@ -24,105 +25,115 @@ func TestIsValidConf(t *testing.T) { wantErr bool }{ { - name: "tc-0", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 0.95, 0.1, 10000}, + name: "should_work", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.95, 0.1, 10000}, wantErr: false, }, { - name: "tc-1", - args: args{"", pathToAnalyticsScript, 4000, 100, 0, 0.95, 0.1, 10000}, + name: "interpreter_empty", + args: args{"", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.95, 0.1, 10000}, wantErr: true, }, { - name: "tc-2", - args: args{"python3", "", 4000, 100, 0, 0.95, 0.1, 10000}, + name: "script_empty", + args: args{"python3", "", 4000, 100, 0, 1.0, 0.95, 0.1, 10000}, wantErr: true, }, { - name: "tc-3", - args: args{"python3", pathToAnalyticsScript, -1, 100, 0, 0.95, 0.1, 10000}, - wantErr: true, // negative cpu + name: "negative_cpu", + args: args{"python3", pathToAnalyticsScript, -1, 100, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-4", - args: args{"python3", pathToAnalyticsScript, 0, 100, 0, 0.95, 0.1, 10000}, - wantErr: true, // zero cpu + name: "zero_cpu", + args: args{"python3", pathToAnalyticsScript, 0, 100, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-5", - args: args{"python3", pathToAnalyticsScript, 999999999, 100, 0, 0.95, 0.1, 10000}, - wantErr: true, // over limit cpu + name: "over_cpu_limit", + args: args{"python3", pathToAnalyticsScript, 999999999, 100, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-6", - args: args{"python3", pathToAnalyticsScript, 4000, -1, 0, 0.95, 0.1, 10000}, - wantErr: true, // negative round base + name: "negative_rounding", + args: args{"python3", pathToAnalyticsScript, 4000, -1, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-7", - args: args{"python3", pathToAnalyticsScript, 4000, 0, 0, 0.95, 0.1, 10000}, - wantErr: true, // zero round base + name: "zero_rounding", + args: args{"python3", pathToAnalyticsScript, 4000, 0, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, + }, + { + name: "over_limit_rounding", + args: args{"python3", pathToAnalyticsScript, 4000, 1001, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, + }, + { + name: "rounding_not_base_10", + args: args{"python3", pathToAnalyticsScript, 4000, 101, 0, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-8", - args: args{"python3", pathToAnalyticsScript, 4000, 1001, 0, 0.95, 0.1, 10000}, - wantErr: true, // over limit round base + name: "negative_proactive", + args: args{"python3", pathToAnalyticsScript, 4000, 100, -1, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-9", - args: args{"python3", pathToAnalyticsScript, 4000, 101, 0, 0.95, 0.1, 10000}, - wantErr: true, // not round base 10 + name: "over_limit_proactive", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 10000, 1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-10", - args: args{"python3", pathToAnalyticsScript, 4000, 100, -1, 0.95, 0.1, 10000}, - wantErr: true, // negative cpu for proactive + name: "boost_factor_to_small", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, -1.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-11", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 10000, 0.95, 0.1, 10000}, - wantErr: true, // over limit cpu for proactive + name: "boost_factor_to_big", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 12.0, 0.95, 0.1, 10000}, + wantErr: true, }, { - name: "tc-12", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, -1.0, 0.1, 10000}, - wantErr: true, // negative value for safeguard + name: "negative_value_for_safeguard", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, -1.0, 0.1, 10000}, + wantErr: true, }, { - name: "tc-13", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 0.0, 0.1, 10000}, - wantErr: true, // zero value for safeguard + name: "zero_limit_safeguard", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.0, 0.1, 10000}, + wantErr: true, }, { - name: "tc-14", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 2.0, 0.1, 10000}, - wantErr: true, // over limit for safeguard + name: "over_limit_safeguard", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 2.0, 0.1, 10000}, + wantErr: true, }, { - name: "tc-15", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 0.95, -1.0, 10000}, - wantErr: true, // negative proactive latency fraction + name: "negative_proactive_latency", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.95, -1.0, 10000}, + wantErr: true, }, { - name: "tc-16", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 0.95, 1.01, 10000}, - wantErr: true, // over limit proactive latency fraction + name: "over_limit_proactive_latency", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.95, 1.01, 10000}, + wantErr: true, }, { - name: "tc-17", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 0.95, 1.0, -1}, - wantErr: true, // negative lookback. + name: "negative_lookback", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.95, 1.0, -1}, + wantErr: true, }, { - name: "tc-18", - args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 0.95, 1.0, 999999}, - wantErr: true, // over limit lookback. + name: "over_limit_lookback", + args: args{"python3", pathToAnalyticsScript, 4000, 100, 0, 1.0, 0.95, 1.0, 999999}, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := isValidConf(tt.args.interpreter, tt.args.script, tt.args.cpuMax, tt.args.cpuRounding, tt.args.maxProActiveCPU, - tt.args.cpuSafeGuardFactor, tt.args.proActiveLatencyPercentage, tt.args.lookBack); (err != nil) != tt.wantErr { + tt.args.boostFactor, tt.args.cpuSafeGuardFactor, tt.args.proActiveLatencyPercentage, tt.args.lookBack); (err != nil) != tt.wantErr { t.Errorf("isValidConf() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/plugins/cpu_scale/cpu-scale-actuator-plugin.yaml b/plugins/cpu_scale/cpu-scale-actuator-plugin.yaml index 54e9731..eaab992 100644 --- a/plugins/cpu_scale/cpu-scale-actuator-plugin.yaml +++ b/plugins/cpu_scale/cpu-scale-actuator-plugin.yaml @@ -10,6 +10,7 @@ data: "cpu_max": 4000, "cpu_rounding": 100, "cpu_safeguard_factor": 0.95, + "boost_factor": 1.0, "look_back": 20, "max_proactive_cpu": 0, "proactive_latency_percentage": 0.1, @@ -30,7 +31,7 @@ spec: serviceAccountName: planner-service-account containers: - name: cpu-scale-actuator - image: 127.0.0.1:5000/cpuscale:0.3.0 + image: 127.0.0.1:5000/cpuscale:0.4.0 imagePullPolicy: Always args: [ "-config", "/config/defaults.json", "-v", "2" ] ports: diff --git a/plugins/energy/Dockerfile b/plugins/energy/Dockerfile new file mode 100644 index 0000000..7c77157 --- /dev/null +++ b/plugins/energy/Dockerfile @@ -0,0 +1,31 @@ +# Copyright (c) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM golang:1.24.6 AS builder + +WORKDIR /plugins + +COPY . ./ + +RUN make prepare-build build-plugins \ + && go run github.com/google/go-licenses@v1.6.0 save "./..." --save_path licenses \ + && hack/additional-licenses.sh + +FROM alpine:3.22 + +RUN adduser -D nonroot +RUN apk add --upgrade --no-cache openssl=~3.5 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.9 \ + py3-pip=~25.1 py3-scikit-learn=~1.5 +RUN pip install --break-system-packages --no-cache-dir pymongo~=4.12 + +WORKDIR /plugins + +COPY --from=builder /plugins/bin/plugins/energy /plugins/bin/plugins/energy +COPY --from=builder /plugins/licenses ./licenses +COPY pkg/planner/actuators/energy/analytics/analytics.py /plugins/pkg/planner/actuators/energy/analytics.py +COPY pkg/planner/actuators/energy/analytics/predict.py /plugins/pkg/planner/actuators/energy/predict.py + +USER nonroot:nonroot +EXPOSE 33334 + +ENTRYPOINT ["/plugins/bin/plugins/energy"] diff --git a/plugins/energy/cmd/energy.go b/plugins/energy/cmd/energy.go new file mode 100644 index 0000000..4b98acc --- /dev/null +++ b/plugins/energy/cmd/energy.go @@ -0,0 +1,99 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/intel/intent-driven-orchestration/pkg/controller" + "github.com/intel/intent-driven-orchestration/pkg/planner/actuators/energy" + pluginsHelper "github.com/intel/intent-driven-orchestration/plugins" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/intel/intent-driven-orchestration/pkg/common" + "k8s.io/klog/v2" +) + +var ( + kubeConfig string + config string +) + +func init() { + flag.StringVar(&kubeConfig, "kubeConfig", "", "Path to a kube config file.") + flag.StringVar(&config, "config", "", "Path to configuration file.") +} + +func main() { + klog.InitFlags(nil) + flag.Parse() + + tmp, err := common.LoadConfig(config, func() interface{} { + return &energy.PowerActuatorConfig{} + }) + if err != nil { + klog.Fatalf("Error loading configuration for actuator: %s", err) + } + cfg := tmp.(*energy.PowerActuatorConfig) + + // validate configuration. + err = pluginsHelper.IsValidGenericConf(cfg.Endpoint, cfg.Port, cfg.PluginManagerEndpoint, cfg.PluginManagerPort, cfg.MongoEndpoint) + if err != nil { + klog.Fatalf("Error on generic configuration for actuator: %s", err) + } + err = isValidConf(cfg.PythonInterpreter, cfg.Prediction, cfg.Analytics, cfg.StepDown, cfg.RenewableLimit, cfg.PowerProfiles) + if err != nil { + klog.Fatalf("Error on configuration for actuator: %s", err) + } + + // get K8s config. + config, err := clientcmd.BuildConfigFromFlags("", kubeConfig) + if err != nil { + klog.Fatalf("Error getting Kubernetes config: %s", err) + } + clusterClient, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Fatalf("Error creating Kubernetes cluster client: %s", err) + } + + // once configuration is ready & valid start the plugin mechanism. + mt := controller.NewMongoTracer(cfg.MongoEndpoint) + actuator := energy.NewPowerActuator(clusterClient, mt, *cfg) + signal := pluginsHelper.StartActuatorPlugin(actuator, cfg.Endpoint, cfg.Port, cfg.PluginManagerEndpoint, cfg.PluginManagerPort) + <-signal +} + +func isValidConf(interpreter, analytics, prediction string, stepDown int, renewableLimit float64, profiles []string) error { + if !pluginsHelper.IsStrConfigValid(interpreter) { + return fmt.Errorf("invalid path to python interpreter: %s", interpreter) + } + + if analytics != "None" { + _, err := os.Stat(analytics) + if err != nil { + return fmt.Errorf("invalid analytics script %s", err) + } + } + + if prediction != "None" { + _, err := os.Stat(prediction) + if err != nil { + return fmt.Errorf("invalid prediction script %s", err) + } + } + + if len(profiles) < 2 { + return fmt.Errorf("invalid number of profiles - expect at least 2: %d", len(profiles)) + } + + if stepDown < 1 || stepDown >= len(profiles) { + return fmt.Errorf("step down needs to be in the range of 1 to %d - was: %d", len(profiles), stepDown) + } + + if renewableLimit <= 0 { + return fmt.Errorf("renewable energy limit needs to be bigger thean 0 - was: %f", renewableLimit) + } + + return nil +} diff --git a/plugins/energy/cmd/energy_test.go b/plugins/energy/cmd/energy_test.go new file mode 100644 index 0000000..3ae12cb --- /dev/null +++ b/plugins/energy/cmd/energy_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "testing" +) + +// pathToAnalyticsScript defines the path to an existing script for this actuator. +const pathToAnalyticsScript = "../../../pkg/planner/actuators/energy/analytics/analytics.py" + +// pathToAnalyticsScript defines the path to an existing script for this actuator. +const pathToPredictScript = "../../../pkg/planner/actuators/energy/analytics/predict.py" + +func TestIsValidConf(t *testing.T) { + tests := []struct { + name string + interpreter string + analyticsScript string + predictionScript string + steps int + renewableLimit float64 + profiles []string + wantErr bool + }{ + { + name: "ExpectNoError", + interpreter: "python3", + analyticsScript: pathToAnalyticsScript, + predictionScript: pathToPredictScript, + steps: 2, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: false, + }, + { + name: "FaultyInterpreter", + interpreter: "", + analyticsScript: pathToAnalyticsScript, + predictionScript: pathToPredictScript, + steps: 2, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: true, + }, + { + name: "NoneShouldBeOkay", + interpreter: "python3", + analyticsScript: "None", + predictionScript: "None", + steps: 2, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: false, + }, + { + name: "FaultyAnalyticsScript", + interpreter: "python3", + analyticsScript: "foo.py", + predictionScript: pathToPredictScript, + steps: 2, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: true, + }, + { + name: "FaultyPredictionScript", + interpreter: "python3", + analyticsScript: pathToAnalyticsScript, + predictionScript: "bar.py", + steps: 2, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: true, + }, + { + name: "StepDownTooSmall", + interpreter: "python3", + analyticsScript: pathToAnalyticsScript, + predictionScript: pathToPredictScript, + steps: 0, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: true, + }, + { + name: "StepDownTooBig", + interpreter: "python3", + analyticsScript: pathToAnalyticsScript, + predictionScript: pathToPredictScript, + steps: 5, + renewableLimit: 0.9, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: true, + }, + { + name: "RenewableLimitTooSmall", + interpreter: "python3", + analyticsScript: pathToAnalyticsScript, + predictionScript: pathToPredictScript, + steps: 1, + renewableLimit: 0.0, + profiles: []string{"shared", "bal-pwr", "bal-perf", "perf"}, + wantErr: true, + }, + { + name: "TooFewProfiles", + interpreter: "python3", + analyticsScript: pathToAnalyticsScript, + predictionScript: pathToPredictScript, + steps: 1, + renewableLimit: 0.1, + profiles: []string{"a"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := isValidConf(tt.interpreter, tt.analyticsScript, tt.predictionScript, tt.steps, tt.renewableLimit, tt.profiles); (err != nil) != tt.wantErr { + t.Errorf("isValidConf() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/plugins/energy/energy-actuator-plugin.yaml b/plugins/energy/energy-actuator-plugin.yaml new file mode 100644 index 0000000..6402263 --- /dev/null +++ b/plugins/energy/energy-actuator-plugin.yaml @@ -0,0 +1,99 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: energy-configmap +data: + defaults.json: |- + { + "interpreter": "python3", + "analytics": "/plugins/pkg/planner/actuators/energy/analytics.py", + "prediction": "/plugins/pkg/planner/actuators/energy/predict.py", + "options": ["None", "power.intel.com/balance-power1", "power.intel.com/balance-performance1", "power.intel.com/performance1"], + "add_proactive_candidates": false, + "renewable_limit": 0.75, + "step_down": 2, + "endpoint": "energy-actuator-service", + "port": 33334, + "mongo_endpoint": "mongodb://planner-mongodb-service:27017/", + "plugin_manager_endpoint": "plugin-manager-service", + "plugin_manager_port": 33333 + } +--- +apiVersion: v1 +kind: Pod +metadata: + name: energy-actuator + labels: + name: energy-actuator +spec: + serviceAccountName: planner-service-account + containers: + - name: energy-actuator + image: 127.0.0.1:5000/energy:0.4.0 + imagePullPolicy: Always + args: [ "-config", "/config/defaults.json" ] + ports: + - containerPort: 33334 + securityContext: + capabilities: + drop: [ 'ALL' ] + seccompProfile: + type: RuntimeDefault + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + resources: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "512Mi" + cpu: "500m" + volumeMounts: + - name: matplotlib-tmp + mountPath: /var/tmp + - name: energy-configmap-volume + mountPath: /config/ + env: + # Needed for the analytics python script. + - name: MONGO_URL + value: "mongodb://planner-mongodb-service:27017/" + - name: MPLCONFIGDIR + value: /var/tmp + volumes: + - name: matplotlib-tmp + emptyDir: + sizeLimit: 100Mi + - name: energy-configmap-volume + configMap: + name: energy-configmap + items: + - key: defaults.json + path: defaults.json + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists +--- +apiVersion: v1 +kind: Service +metadata: + name: energy-actuator-service +spec: + clusterIP: None + selector: + name: energy-actuator + ports: + - protocol: TCP + port: 33334 + targetPort: 33334 diff --git a/plugins/rdt/Dockerfile b/plugins/rdt/Dockerfile index 5bbe3b9..d81fbdc 100644 --- a/plugins/rdt/Dockerfile +++ b/plugins/rdt/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.24.2 AS builder +FROM golang:1.24.6 AS builder WORKDIR /plugins @@ -11,19 +11,19 @@ RUN make prepare-build build-plugins \ && go run github.com/google/go-licenses@v1.6.0 save "./..." --save_path licenses \ && hack/additional-licenses.sh -FROM alpine:3.20 +FROM alpine:3.22 RUN adduser -D nonroot -RUN apk add --upgrade --no-cache openssl=~3.3 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.7 \ - py3-pip=~24.0 py3-scikit-learn=~1.3 -RUN pip install --break-system-packages --no-cache-dir pymongo~=4.6 +RUN apk add --upgrade --no-cache openssl=~3.5 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.9 \ + py3-pip=~25.1 py3-scikit-learn=~1.5 +RUN pip install --break-system-packages --no-cache-dir pymongo~=4.12 WORKDIR /plugins COPY --from=builder /plugins/bin/plugins/rdt /plugins/bin/plugins/rdt COPY --from=builder /plugins/licenses ./licenses -COPY pkg/planner/actuators/platform/analyze.py /plugins/pkg/planner/actuators/platform/analyze.py -COPY pkg/planner/actuators/platform/predict.py /plugins/pkg/planner/actuators/platform/predict.py +COPY pkg/planner/actuators/platform/analytics/rdt_effect.py /plugins/pkg/planner/actuators/platform/rdt_effect.py +COPY pkg/planner/actuators/platform/analytics/rdt_predict.py /plugins/pkg/planner/actuators/platform/rdt_predict.py USER nonroot:nonroot EXPOSE 33334 diff --git a/plugins/rdt/cmd/rdt_test.go b/plugins/rdt/cmd/rdt_test.go index 7e19aa2..aa86b75 100644 --- a/plugins/rdt/cmd/rdt_test.go +++ b/plugins/rdt/cmd/rdt_test.go @@ -4,8 +4,11 @@ import ( "testing" ) -// pathToAnalyticsScript defines the path to an existing script for this actuator. -const pathToAnalyticsScript = "../../../pkg/planner/actuators/platform/analyze.py" +// pathToAnalyticsScript defines the path to an existing analytics script for this actuator. +const pathToAnalyticsScript = "../../../pkg/planner/actuators/platform/analytics/rdt_effect.py" + +// pathToPredictScript defines the path to an existing prediction script for this actuator. +const pathToPredictScript = "../../../pkg/planner/actuators/platform/analytics/rdt_predict.py" func TestIsValidConf(t *testing.T) { type args struct { @@ -21,17 +24,17 @@ func TestIsValidConf(t *testing.T) { }{ { name: "tc-0", - args: args{"python3", pathToAnalyticsScript, "../../../pkg/planner/actuators/platform/predict.py", []string{"cos0", "cos1"}}, + args: args{"python3", pathToAnalyticsScript, pathToPredictScript, []string{"cos0", "cos1"}}, wantErr: false, }, { name: "tc-1", - args: args{"", pathToAnalyticsScript, "../../../pkg/planner/actuators/platform/predict.py", []string{"cos0", "cos1"}}, + args: args{"", pathToAnalyticsScript, pathToPredictScript, []string{"cos0", "cos1"}}, wantErr: true, // wrong interpreter }, { name: "tc-2", - args: args{"python3", "", "../../../pkg/planner/actuators/platform/predict.py", []string{"cos0", "cos1"}}, + args: args{"python3", "", pathToPredictScript, []string{"cos0", "cos1"}}, wantErr: true, // invalid analytics script }, { @@ -41,7 +44,7 @@ func TestIsValidConf(t *testing.T) { }, { name: "tc-4", - args: args{"python3", pathToAnalyticsScript, "../../../pkg/planner/actuators/platform/predict.py", []string{}}, + args: args{"python3", pathToAnalyticsScript, pathToPredictScript, []string{}}, wantErr: true, // invalid cos options }, } diff --git a/plugins/rdt/rdt-actuator-plugin.yaml b/plugins/rdt/rdt-actuator-plugin.yaml index ba6b2b7..b2720fa 100644 --- a/plugins/rdt/rdt-actuator-plugin.yaml +++ b/plugins/rdt/rdt-actuator-plugin.yaml @@ -6,8 +6,8 @@ data: defaults.json: |- { "interpreter": "python3", - "analytics_script": "./pkg/planner/actuators/platform/analyze.py", - "prediction_script": "./pkg/planner/actuators/platform/predict.py", + "analytics_script": "./pkg/planner/actuators/platform/rdt_effect.py", + "prediction_script": "./pkg/planner/actuators/platform/rdt_predict.py", "options": [ "None", "COS1", @@ -30,7 +30,8 @@ spec: serviceAccountName: planner-service-account containers: - name: rdt-actuator - image: 127.0.0.1:5000/rdt:0.3.0 + image: 127.0.0.1:5000/rdt:0.4.0 + imagePullPolicy: Always args: [ "-config", "/config/defaults.json" ] ports: - containerPort: 33334 diff --git a/plugins/rm_pod/Dockerfile b/plugins/rm_pod/Dockerfile index 557b14a..30cbb18 100644 --- a/plugins/rm_pod/Dockerfile +++ b/plugins/rm_pod/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.24.2 AS builder +FROM golang:1.24.6 AS builder WORKDIR /plugins diff --git a/plugins/rm_pod/rmpod-actuator-plugin.yaml b/plugins/rm_pod/rmpod-actuator-plugin.yaml index 8a19aa2..249289a 100644 --- a/plugins/rm_pod/rmpod-actuator-plugin.yaml +++ b/plugins/rm_pod/rmpod-actuator-plugin.yaml @@ -24,7 +24,8 @@ spec: serviceAccountName: planner-service-account containers: - name: rmpod-actuator - image: 127.0.0.1:5000/rmpod:0.3.0 + image: 127.0.0.1:5000/rmpod:0.4.0 + imagePullPolicy: Always args: [ "-config", "/config/defaults.json" ] ports: - containerPort: 33334 diff --git a/plugins/scale_out/Dockerfile b/plugins/scale_out/Dockerfile index 7ecc890..66c4f3d 100644 --- a/plugins/scale_out/Dockerfile +++ b/plugins/scale_out/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -FROM golang:1.24.2 AS builder +FROM golang:1.24.6 AS builder WORKDIR /plugins @@ -11,12 +11,12 @@ RUN make prepare-build build-plugins \ && go run github.com/google/go-licenses@v1.6.0 save "./..." --save_path licenses \ && hack/additional-licenses.sh -FROM alpine:3.20 +FROM alpine:3.22 RUN adduser -D nonroot -RUN apk add --upgrade --no-cache openssl=~3.3 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.7 \ - py3-pip=~24.0 py3-scikit-learn=~1.3 -RUN pip install --break-system-packages --no-cache-dir pymongo~=4.6 +RUN apk add --upgrade --no-cache openssl=~3.5 && apk add --no-cache python3=~3.12 py3-matplotlib=~3.9 \ + py3-pip=~25.1 py3-scikit-learn=~1.5 +RUN pip install --break-system-packages --no-cache-dir pymongo~=4.12 WORKDIR /plugins diff --git a/plugins/scale_out/scaleout-actuator-plugin.yaml b/plugins/scale_out/scaleout-actuator-plugin.yaml index 9471fa6..23be5ef 100644 --- a/plugins/scale_out/scaleout-actuator-plugin.yaml +++ b/plugins/scale_out/scaleout-actuator-plugin.yaml @@ -28,7 +28,8 @@ spec: serviceAccountName: planner-service-account containers: - name: scaleout-actuator - image: 127.0.0.1:5000/scaleout:0.3.0 + image: 127.0.0.1:5000/scaleout:0.4.0 + imagePullPolicy: Always args: [ "-config", "/config/defaults.json" ] ports: - containerPort: 33334