diff --git a/Makefile b/Makefile index 48908d3f..cc4d3a11 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ ifeq ($(CGO_APPS), 1) PROMU_CONF ?= .promu/.promu-cgo.yml pkgs := ./pkg/sqlite3 ./pkg/api/cli \ ./pkg/api/db ./pkg/api/db/migrator ./pkg/api/helper \ - ./pkg/api/resource ./pkg/api/resource/slurm ./pkg/api/resource/openstack \ + ./pkg/api/resource ./pkg/api/resource/slurm ./pkg/api/resource/openstack ./pkg/api/resource/k8s \ ./pkg/api/updater ./pkg/api/updater/tsdb \ ./pkg/api/http ./cmd/ceems_api_server \ ./pkg/lb/backend ./pkg/lb/cli \ @@ -197,21 +197,26 @@ test-e2e: $(PROMTOOL) build pkg/collector/testdata/sys/.unpacked pkg/collector/t ./scripts/e2e-test.sh -s api-project-query ./scripts/e2e-test.sh -s api-project-empty-query ./scripts/e2e-test.sh -s api-project-admin-query + ./scripts/e2e-test.sh -s api-project-query-k8s ./scripts/e2e-test.sh -s api-user-query ./scripts/e2e-test.sh -s api-user-admin-query + ./scripts/e2e-test.sh -s api-user-query-k8s ./scripts/e2e-test.sh -s api-cluster-admin-query ./scripts/e2e-test.sh -s api-uuid-query ./scripts/e2e-test.sh -s api-running-query + ./scripts/e2e-test.sh -s api-units-query-k8s ./scripts/e2e-test.sh -s api-admin-query ./scripts/e2e-test.sh -s api-admin-query-all ./scripts/e2e-test.sh -s api-admin-query-all-selected-fields ./scripts/e2e-test.sh -s api-admin-denied-query ./scripts/e2e-test.sh -s api-current-usage-query ./scripts/e2e-test.sh -s api-current-usage-experimental-query - ./scripts/e2e-test.sh -s api-global-usage-query ./scripts/e2e-test.sh -s api-current-usage-admin-query ./scripts/e2e-test.sh -s api-current-usage-admin-experimental-query + ./scripts/e2e-test.sh -s api-current-usage-query-k8s + ./scripts/e2e-test.sh -s api-global-usage-query ./scripts/e2e-test.sh -s api-global-usage-admin-query + ./scripts/e2e-test.sh -s api-global-usage-query-k8s ./scripts/e2e-test.sh -s api-current-usage-admin-denied-query ./scripts/e2e-test.sh -s api-current-stats-admin-query ./scripts/e2e-test.sh -s api-global-stats-admin-query @@ -277,21 +282,26 @@ test-e2e-update: $(PROMTOOL) build pkg/collector/testdata/sys/.unpacked pkg/coll ./scripts/e2e-test.sh -s api-project-query -u || true ./scripts/e2e-test.sh -s api-project-empty-query -u || true ./scripts/e2e-test.sh -s api-project-admin-query -u || true + ./scripts/e2e-test.sh -s api-project-query-k8s -u || true ./scripts/e2e-test.sh -s api-user-query -u || true ./scripts/e2e-test.sh -s api-user-admin-query -u || true + ./scripts/e2e-test.sh -s api-user-query-k8s -u || true ./scripts/e2e-test.sh -s api-cluster-admin-query -u || true ./scripts/e2e-test.sh -s api-uuid-query -u || true ./scripts/e2e-test.sh -s api-running-query -u || true + ./scripts/e2e-test.sh -s api-units-query-k8s -u || true ./scripts/e2e-test.sh -s api-admin-query -u || true ./scripts/e2e-test.sh -s api-admin-query-all -u || true ./scripts/e2e-test.sh -s api-admin-query-all-selected-fields -u || true ./scripts/e2e-test.sh -s api-admin-denied-query -u || true ./scripts/e2e-test.sh -s api-current-usage-query -u || true ./scripts/e2e-test.sh -s api-current-usage-experimental-query -u || true - ./scripts/e2e-test.sh -s api-global-usage-query -u || true ./scripts/e2e-test.sh -s api-current-usage-admin-query -u || true ./scripts/e2e-test.sh -s api-current-usage-admin-experimental-query -u || true + ./scripts/e2e-test.sh -s api-current-usage-query-k8s -u || true + ./scripts/e2e-test.sh -s api-global-usage-query -u || true ./scripts/e2e-test.sh -s api-global-usage-admin-query -u || true + ./scripts/e2e-test.sh -s api-global-usage-query-k8s -u || true ./scripts/e2e-test.sh -s api-current-usage-admin-denied-query -u || true ./scripts/e2e-test.sh -s api-current-stats-admin-query -u || true ./scripts/e2e-test.sh -s api-global-stats-admin-query -u || true diff --git a/README.md b/README.md index 2f24a3b6..8edca0ab 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ managers (SLURM, Openstack, k8s) - Supports zero instrumentation eBPF based continuous profiling using [Grafana Pyroscope](https://grafana.com/oss/pyroscope/) as backend - Realtime access to metrics *via* Grafana dashboards or a simple CLI tool -- Access control to Prometheus and Pyroscope datasources in Grafana +- Multi-tenancy and access control to Prometheus and Pyroscope datasources in Grafana - Stores aggregated metrics in a separate DB that can be retained for long time - CEEMS apps are [capability aware](https://tbhaxor.com/understanding-linux-capabilities/) diff --git a/build/config/ceems_api_server/ceems_api_server.yml b/build/config/ceems_api_server/ceems_api_server.yml index 3fbe7cd5..428267b6 100644 --- a/build/config/ceems_api_server/ceems_api_server.yml +++ b/build/config/ceems_api_server/ceems_api_server.yml @@ -231,8 +231,10 @@ clusters: [] # # # - id: default - # # Resource manager of the cluster. Currently only `slurm` is supported. In future, - # # `openstack` will be supported + # # Resource manager of the cluster. Currently supported managers: + # # - `slurm` + # # - `openstack` + # # - `k8s` # # # manager: slurm @@ -437,28 +439,69 @@ clusters: [] # # Any other configuration needed to reach API server of the resource manager # # can be configured in this section. # # - # # Currently this section is used for Openstack resource manager - # # to configure API versions - # # - # # In the case of Openstack, this section must have two keys `api_service_endpoints` - # # and `auth`. Both of these are compulsory. - # # `api_service_endpoints` must provide API endpoints for compute and identity - # # services as provided in service catalog of Openstack cluster. `auth` must be the - # # same `auth` object that must be sent in POST request to keystone to get a API token. + # # Currently this section is used for Openstack and k8s resource managers + # # to configure API servers # # # extra_config: {} - # # api_service_endpoints: - # # compute: https://openstack-nova.example.com/v2.1 - # # identity: https://openstack-keystone.example.com - # # auth: - # # identity: - # # methods: - # # - password - # # password: - # # user: - # # name: admin - # # password: supersecret - + # # # In the case of Openstack, this section must have two keys `api_service_endpoints` + # # # and `auth`. Both of these are compulsory. + # # # `api_service_endpoints` must provide API endpoints for compute and identity + # # # services as provided in service catalog of Openstack cluster. `auth` must be the + # # # same `auth` object that must be sent in POST request to keystone to get a API token. + # # # + # # api_service_endpoints: + # # compute: https://openstack-nova.example.com/v2.1 + # # identity: https://openstack-keystone.example.com + # # auth: + # # identity: + # # methods: + # # - password + # # password: + # # user: + # # name: admin + # # password: supersecret + + # # # In the case of k8s, this section is used to configure the k8s API related config + # # # + # # # Path to the kube config file when out-of-cluster config file is used. If the + # # # ceems_api_server is running in a pod, in-cluster config will be used by default. + # # # + # # kubeconfig_file: '' + # # # + # # # List of annotation names where the name of the user that created/modified the pod is stored. + # # # The search for username will be stopped when the first annotation is found in the list. + # # # Therefore the order of the list is important. + # # # + # # # Default: [ceems.io/created-by] + # # # + # # username_annotations: [] + # # # + # # # List of annotation names where the name of the project that the pod belongs to is stored. + # # # By default the project is always set to the namespace of the pod and when atleast one of + # # # annotation names are found in the pod spec, the namespace will be overridded by the value + # # # of the annotation + # # # + # # project_annotations: [] + # # # + # # # List of GPU resource names in the cluster. + # # # + # # # Default: [nvidia.com/gpu, amd.com/gpu] + # # # + # # gpu_resource_names: [] + # # # + # # # Path to the file that contains the list of namespaces and allowed users in each namespace. + # # # The format of the file must be as follows: + # # # + # # # users: + # # # ns1: + # # # - usr1 + # # # - usr2 + # # # ns2: + # # # - usr1 + # # # - usr3 + # # # + # # ns_users_list_file: '' + # A list of Updaters that will be used to update the compute unit metrics. This update # step can be used to update the aggregate metrics of each compute unit in real time # or to add complementary information to the compute units from on-premise third diff --git a/cmd/cacct/main.go b/cmd/cacct/main.go index b333446c..cf7b3d2c 100644 --- a/cmd/cacct/main.go +++ b/cmd/cacct/main.go @@ -8,7 +8,6 @@ import ( "os/user" "path/filepath" "slices" - "sort" "strings" "time" @@ -237,12 +236,12 @@ type field struct { } // titles return header titles for the table. -func (f field) titles() []interface{} { +func (f field) titles() []any { if len(f.keys) <= 1 { - return []interface{}{f.title} + return []any{f.title} } - t := make([]interface{}, len(f.keys)) + t := make([]any, len(f.keys)) for i := range len(f.keys) { t[i] = f.title } @@ -251,12 +250,12 @@ func (f field) titles() []interface{} { } // subtitles return header subtitles for the table. -func (f field) subtitles() []interface{} { +func (f field) subtitles() []any { if len(f.keys) <= 1 { - return []interface{}{""} + return []any{""} } - t := make([]interface{}, len(f.keys)) + t := make([]any, len(f.keys)) for i, k := range f.keys { t[i] = k } @@ -294,7 +293,7 @@ type Config struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *Config) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *c = Config{} c.API.UserHeaderName = "X-Grafana-User" @@ -316,7 +315,7 @@ type WebConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (w *WebConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (w *WebConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *w = WebConfig{} w.HTTPClientConfig = http_config.DefaultHTTPClientConfig @@ -726,7 +725,7 @@ func sortedKeys[K cmp.Ordered, V any](m map[K]V) []K { i++ } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + slices.Sort(keys) return keys } @@ -734,7 +733,7 @@ func sortedKeys[K cmp.Ordered, V any](m map[K]V) []K { func splitString(s, d string) []string { //nolint:unparam var parts []string - for _, p := range strings.Split(s, d) { + for p := range strings.SplitSeq(s, d) { if p != "" { parts = append(parts, p) } diff --git a/cmd/ceems_api_server/main.go b/cmd/ceems_api_server/main.go index 51613987..2cfa4a1f 100644 --- a/cmd/ceems_api_server/main.go +++ b/cmd/ceems_api_server/main.go @@ -9,6 +9,7 @@ import ( "os" "github.com/mahendrapaipuri/ceems/pkg/api/cli" + _ "github.com/mahendrapaipuri/ceems/pkg/api/resource/k8s" _ "github.com/mahendrapaipuri/ceems/pkg/api/resource/openstack" _ "github.com/mahendrapaipuri/ceems/pkg/api/resource/slurm" _ "github.com/mahendrapaipuri/ceems/pkg/api/updater/tsdb" diff --git a/cmd/ceems_k8s_admission_controller/base/patch.go b/cmd/ceems_k8s_admission_controller/base/patch.go index b8f74a32..082e8681 100644 --- a/cmd/ceems_k8s_admission_controller/base/patch.go +++ b/cmd/ceems_k8s_admission_controller/base/patch.go @@ -10,14 +10,14 @@ const ( // PatchOperation is an operation of a JSON patch https://tools.ietf.org/html/rfc6902. type PatchOperation struct { - Op string `json:"op"` - Path string `json:"path"` - From string `json:"from"` - Value interface{} `json:"value,omitempty"` + Op string `json:"op"` + Path string `json:"path"` + From string `json:"from"` + Value any `json:"value,omitempty"` } // AddPatchOperation returns an add JSON patch operation. -func AddPatchOperation(path string, value interface{}) PatchOperation { +func AddPatchOperation(path string, value any) PatchOperation { return PatchOperation{ Op: addOperation, Path: path, @@ -34,7 +34,7 @@ func RemovePatchOperation(path string) PatchOperation { } // ReplacePatchOperation returns a replace JSON patch operation. -func ReplacePatchOperation(path string, value interface{}) PatchOperation { +func ReplacePatchOperation(path string, value any) PatchOperation { return PatchOperation{ Op: replaceOperation, Path: path, diff --git a/cmd/ceems_k8s_admission_controller/http/handlers.go b/cmd/ceems_k8s_admission_controller/http/handlers.go index 8e0e60df..4eeda9a4 100644 --- a/cmd/ceems_k8s_admission_controller/http/handlers.go +++ b/cmd/ceems_k8s_admission_controller/http/handlers.go @@ -85,7 +85,7 @@ func (h *admissionHandler) Serve(hook base.Hook) http.HandlerFunc { var result *base.Result - var operation interface{} + var operation any var version string diff --git a/cmd/ceems_tool/testdata/output/e2e-test-recording-rules-output.txt b/cmd/ceems_tool/testdata/output/e2e-test-recording-rules-output.txt index 7b1b1269..d571e421 100644 --- a/cmd/ceems_tool/testdata/output/e2e-test-recording-rules-output.txt +++ b/cmd/ceems_tool/testdata/output/e2e-test-recording-rules-output.txt @@ -1595,7 +1595,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average GPU, GPU memory usages and # total GPU power and its equivalent emissions aggregared for all hosts @@ -1632,7 +1632,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_gpu_power_watts:pue{job="amd-device-metrics-gpu"} / 3.6e+06, + 44.179085 * job:ceems_gpu_power_watts:pue{job="amd-device-metrics-gpu"} / 3.6e+06, "provider", "owid", "instance", @@ -1751,7 +1751,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average CPU, CPU memory usages and # total host power (excluding GPUs) and its equivalent emissions aggregared for all hosts @@ -1812,7 +1812,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_host_power_watts:pue{job="cpu-cray-amd-gpu"} / 3.6e+06, + 44.179085 * job:ceems_host_power_watts:pue{job="cpu-cray-amd-gpu"} / 3.6e+06, "provider", "owid", "instance", @@ -1930,7 +1930,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average CPU, CPU memory usages and # total host power (excluding GPUs) and its equivalent emissions aggregared for all hosts @@ -1983,7 +1983,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_host_power_watts:pue{job="cpu-ipmi-nvidia-gpu"} / 3.6e+06, + 44.179085 * job:ceems_host_power_watts:pue{job="cpu-ipmi-nvidia-gpu"} / 3.6e+06, "provider", "owid", "instance", @@ -2132,7 +2132,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average CPU, CPU memory usages and # total host power (excluding GPUs) and its equivalent emissions aggregared for all hosts @@ -2185,7 +2185,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_host_power_watts:pue{job="cpu-only-ipmi"} / 3.6e+06, + 44.179085 * job:ceems_host_power_watts:pue{job="cpu-only-ipmi"} / 3.6e+06, "provider", "owid", "instance", @@ -2291,7 +2291,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average CPU, CPU memory usages and # total host power (excluding GPUs) and its equivalent emissions aggregared for all hosts @@ -2352,7 +2352,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_host_power_watts:pue{job="cpu-only-rapl"} / 3.6e+06, + 44.179085 * job:ceems_host_power_watts:pue{job="cpu-only-rapl"} / 3.6e+06, "provider", "owid", "instance", @@ -2501,7 +2501,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average CPU, CPU memory usages and # total host power (excluding GPUs) and its equivalent emissions aggregared for all hosts @@ -2554,7 +2554,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_host_power_watts:pue{job="cpu-only-redfish"} / 3.6e+06, + 44.179085 * job:ceems_host_power_watts:pue{job="cpu-only-redfish"} / 3.6e+06, "provider", "owid", "instance", @@ -2703,7 +2703,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # The following recording rules estimate the average CPU, CPU memory usages and # total host power (excluding GPUs) and its equivalent emissions aggregared for all hosts @@ -2756,7 +2756,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_host_power_watts:pue{job="cpu-redfish-nvidia-gpu"} / 3.6e+06, + 44.179085 * job:ceems_host_power_watts:pue{job="cpu-redfish-nvidia-gpu"} / 3.6e+06, "provider", "owid", "instance", @@ -2859,7 +2859,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # Profiling metrics @@ -2916,7 +2916,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_gpu_power_watts:pue{job="ipmi-nvidia-gpu"} / 3.6e+06, + 44.179085 * job:ceems_gpu_power_watts:pue{job="ipmi-nvidia-gpu"} / 3.6e+06, "provider", "owid", "instance", @@ -3019,7 +3019,7 @@ groups: "instance", "(.*)" ) - * 56.03859 + * 44.179085 # Profiling metrics @@ -3076,7 +3076,7 @@ groups: expr: |2 label_replace( label_replace( - 56.03859 * job:ceems_gpu_power_watts:pue{job="nvidia-gpu"} / 3.6e+06, + 44.179085 * job:ceems_gpu_power_watts:pue{job="nvidia-gpu"} / 3.6e+06, "provider", "owid", "instance", diff --git a/cmd/redfish_proxy/main.go b/cmd/redfish_proxy/main.go index 1efd761b..d94ffb0c 100644 --- a/cmd/redfish_proxy/main.go +++ b/cmd/redfish_proxy/main.go @@ -51,7 +51,7 @@ type Target struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (t *Target) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *Target) UnmarshalYAML(unmarshal func(any) error) error { var tmp struct { HostAddrs []string `yaml:"host_ip_addrs"` URL string `yaml:"url"` @@ -100,7 +100,7 @@ type ProxyConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (r *ProxyConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (r *ProxyConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *r = ProxyConfig{} r.AllowedAPIResources = defaultAllowedAPIResources @@ -139,7 +139,7 @@ type Redfish struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (r *Redfish) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (r *Redfish) UnmarshalYAML(unmarshal func(any) error) error { type plain Redfish if err := unmarshal((*plain)(r)); err != nil { diff --git a/go.mod b/go.mod index 76e8c87e..035ede7f 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.24.2 require ( connectrpc.com/connect v1.18.1 github.com/alecthomas/kingpin/v2 v2.4.0 + github.com/ceems-dev/perf-utils v0.0.0-20250716113832-99ba71e88ef4 github.com/cilium/ebpf v0.19.0 github.com/containerd/cgroups/v3 v3.0.5 github.com/go-chi/httprate v0.15.0 @@ -18,7 +19,6 @@ require ( github.com/grafana/pyroscope/ebpf v0.4.11 github.com/jedib0t/go-pretty/v6 v6.6.7 github.com/jellydator/ttlcache/v3 v3.4.0 - github.com/mahendrapaipuri/perf-utils v0.0.0-20241102115757-6c72709e1c07 github.com/mattn/go-sqlite3 v1.14.28 github.com/prometheus/client_golang v1.22.0 github.com/prometheus/common v0.65.0 @@ -32,7 +32,7 @@ require ( github.com/wneessen/go-fileperm v0.2.1 github.com/zeebo/xxh3 v1.0.2 golang.org/x/crypto v0.39.0 - golang.org/x/sys v0.33.0 + golang.org/x/sys v0.34.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 15943edd..ce9e800d 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270 h1:JIxGEMs4E5 github.com/avvmoto/buf-readerat v0.0.0-20171115124131-a17c8cb89270/go.mod h1:2XtVRGCw/HthOLxU0Qw6o6jSJrcEoOb2OCCl8gQYvGw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/ceems-dev/perf-utils v0.0.0-20250716113832-99ba71e88ef4 h1:mchfmIweUjhXxH/LER0/NvqzkjIhCINfmlBYJH5lJXw= +github.com/ceems-dev/perf-utils v0.0.0-20250716113832-99ba71e88ef4/go.mod h1:6yFz1mwaJvVpwXLJW3oS82o2oe0oSVXuKA0gi9FBrSY= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.19.0 h1:Ro/rE64RmFBeA9FGjcTc+KmCeY6jXmryu6FfnzPRIao= @@ -120,8 +122,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mahendrapaipuri/perf-utils v0.0.0-20241102115757-6c72709e1c07 h1:FssguDihSIG+XLpvLVNXNr8AfoSwTrNaAqWsbYTA1sE= -github.com/mahendrapaipuri/perf-utils v0.0.0-20241102115757-6c72709e1c07/go.mod h1:B3EnGJVDIBJ9PRO9cngEsv/p1nmOgfxB2eKS8yhbrSM= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= @@ -272,8 +272,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211031064116-611d5d643895/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/internal/common/helpers.go b/internal/common/helpers.go index ff03e4ed..b278c4d5 100644 --- a/internal/common/helpers.go +++ b/internal/common/helpers.go @@ -94,7 +94,7 @@ func replaceNodelistDelimiter(nodelistExp string) string { func expandNodelist(nodelistExp string) []string { var nodeNames []string // First split by | to get individual nodes - for _, nodeexp := range strings.Split(nodelistExp, "|") { + for nodeexp := range strings.SplitSeq(nodelistExp, "|") { if strings.Contains(nodeexp, "[") { matches := nodelistRegExp.FindAllString(nodeexp, -1) if len(matches) == 0 { @@ -106,7 +106,7 @@ func expandNodelist(nodelistExp string) []string { matchSansBrackets := match[1 : len(match)-1] // matchSansBranckets can have multiple ranges like 0-2,4,5-8 // Split them by "," - for _, subMatches := range strings.Split(matchSansBrackets, ",") { + for subMatches := range strings.SplitSeq(matchSansBrackets, ",") { subMatch := strings.Split(subMatches, "-") // If subMatch is single number, copy it as second index if len(subMatch) == 1 { @@ -203,10 +203,10 @@ func GetUUIDFromString(stringSlice []string) (string, error) { // fmt.Sprint() with default formatting is used to convert the key to a string key. // // Nicked from https://github.com/icza/dyno -func ConvertMapI2MapS(v interface{}) interface{} { +func ConvertMapI2MapS(v any) any { switch x := v.(type) { - case map[interface{}]interface{}: - m := map[string]interface{}{} + case map[any]any: + m := map[string]any{} for k, v2 := range x { switch k2 := k.(type) { @@ -219,12 +219,12 @@ func ConvertMapI2MapS(v interface{}) interface{} { v = m - case []interface{}: + case []any: for i, v2 := range x { x[i] = ConvertMapI2MapS(v2) } - case map[string]interface{}: + case map[string]any: for k, v2 := range x { x[k] = ConvertMapI2MapS(v2) } diff --git a/internal/common/helpers_test.go b/internal/common/helpers_test.go index edf4efa1..474644d8 100644 --- a/internal/common/helpers_test.go +++ b/internal/common/helpers_test.go @@ -273,9 +273,9 @@ func TestGetUuid(t *testing.T) { func TestConvertMapI2MapS(t *testing.T) { cases := []struct { - title string // Title of the test case - v interface{} // Input dynamic object - exp interface{} // Expected result + title string // Title of the test case + v any // Input dynamic object + exp any // Expected result }{ { title: "nil value", @@ -289,44 +289,44 @@ func TestConvertMapI2MapS(t *testing.T) { }, { title: "map[interfac{}]interface{} value", - v: map[interface{}]interface{}{ + v: map[any]any{ "s": "s", 1: 1, }, - exp: map[string]interface{}{ + exp: map[string]any{ "s": "s", "1": 1, }, }, { title: "nested maps and slices", - v: map[interface{}]interface{}{ + v: map[any]any{ "s": "s", 1: 1, - float64(0): []interface{}{ + float64(0): []any{ 1, "x", - map[interface{}]interface{}{ + map[any]any{ "s": "s", 2.0: 2, }, - map[string]interface{}{ + map[string]any{ "s": "s", "1": 1, }, }, }, - exp: map[string]interface{}{ + exp: map[string]any{ "s": "s", "1": 1, - "0": []interface{}{ + "0": []any{ 1, "x", - map[string]interface{}{ + map[string]any{ "s": "s", "2": 2, }, - map[string]interface{}{ + map[string]any{ "s": "s", "1": 1, }, diff --git a/internal/security/exec.go b/internal/security/exec.go index 72056518..aa55544b 100644 --- a/internal/security/exec.go +++ b/internal/security/exec.go @@ -19,7 +19,7 @@ var ( type SCConfig struct { Logger *slog.Logger - Func func(interface{}) error + Func func(any) error Caps []cap.Value Name string @@ -34,7 +34,7 @@ type SCConfig struct { type SecurityContext struct { logger *slog.Logger launcher *cap.Launcher - f func(interface{}) error + f func(any) error caps []cap.Value capSet *cap.Set execNatively bool @@ -60,7 +60,7 @@ func NewSecurityContext(c *SCConfig) (*SecurityContext, error) { } // Exec executes the function inside the security context and returns error if any. -func (s *SecurityContext) Exec(data interface{}) error { +func (s *SecurityContext) Exec(data any) error { // If ExecNatively is set to true, execute function natively without creating // a security context if s.execNatively { @@ -117,7 +117,7 @@ func (s *SecurityContext) dropCaps() error { // targetFunc is the function that will be executed in the security context. The passed // function is embedded between raising and dropping capabilities so that the function // gets appropriate capabilities during its execution. -func (s *SecurityContext) targetFunc(data interface{}) error { +func (s *SecurityContext) targetFunc(data any) error { // First raise all necessary capabilities // Ignore all errors as any missing capabilities will fail // the main function. @@ -162,7 +162,7 @@ type ExecSecurityCtxData struct { } // ExecAsUser executes a subprocess as a given user inside a security context. -func ExecAsUser(data interface{}) error { +func ExecAsUser(data any) error { // Assert data type var ctxData *ExecSecurityCtxData diff --git a/internal/security/exec_test.go b/internal/security/exec_test.go index 151f8b40..b5856c21 100644 --- a/internal/security/exec_test.go +++ b/internal/security/exec_test.go @@ -18,7 +18,7 @@ type testData struct { gotID string } -func testFunc(d interface{}) error { +func testFunc(d any) error { data, ok := d.(*testData) if !ok { return fmt.Errorf("cannot be asserted: %v", d) diff --git a/internal/security/manager.go b/internal/security/manager.go index 5445ff6b..a2e4c821 100644 --- a/internal/security/manager.go +++ b/internal/security/manager.go @@ -452,7 +452,7 @@ func hasReadWriteExecutable(p fileperm.PermUser, currentUser *user.User, runAsUs } // deleteACLEntries deletes ACL entries inside a security context. -func deleteACLEntries(data interface{}) error { +func deleteACLEntries(data any) error { // Assert data is of slurmSecurityCtxData var d *deleteACLEntriesCtxData diff --git a/internal/structset/structset.go b/internal/structset/structset.go index bb4548b2..a8117896 100644 --- a/internal/structset/structset.go +++ b/internal/structset/structset.go @@ -11,7 +11,7 @@ import ( var fieldIndexesCache sync.Map // StructFieldNames returns all fields in a given struct. -func StructFieldNames(s interface{}) []string { +func StructFieldNames(s any) []string { v := reflect.ValueOf(s) typeOfS := v.Type() @@ -50,7 +50,7 @@ func tagValue(field reflect.StructField, tag string) string { } // StructFieldTagValues returns all tag names in a given struct for a given tag. -func StructFieldTagValues(s interface{}, tag string) []string { +func StructFieldTagValues(s any, tag string) []string { v := reflect.ValueOf(s) typeOfS := v.Type() @@ -66,7 +66,7 @@ func StructFieldTagValues(s interface{}, tag string) []string { } // StructFieldTagMap returns a map of tags using keyTag as map key and valueTag as map value. -func StructFieldTagMap(s interface{}, keyTag string, valueTag string) map[string]string { +func StructFieldTagMap(s any, keyTag string, valueTag string) map[string]string { v := reflect.ValueOf(s) typeOfS := v.Type() diff --git a/internal/structset/structset_test.go b/internal/structset/structset_test.go index 25db1727..06ee3b2b 100644 --- a/internal/structset/structset_test.go +++ b/internal/structset/structset_test.go @@ -9,11 +9,11 @@ import ( // testStruct is a test struct that will be used in tests. type testStruct struct { - ID int `json:"-" sql:"id"` - Field1 string `json:"field1,omitempty" sql:"f1"` - Field2 bool `json:"field2" sql:"f2"` - Field3 interface{} `sql:"f3"` - Field4 []string `json:"field4" sql:"f4"` + ID int `json:"-" sql:"id"` + Field1 string `json:"field1,omitempty" sql:"f1"` + Field2 bool `json:"field2" sql:"f2"` + Field3 any `sql:"f3"` + Field4 []string `json:"field4" sql:"f4"` } func TestStructFieldNames(t *testing.T) { @@ -51,7 +51,7 @@ func TestCachedFiledIndexes(t *testing.T) { // Get length of sync map var i int - fieldIndexesCache.Range(func(k, v interface{}) bool { + fieldIndexesCache.Range(func(k, v any) bool { i++ return true diff --git a/pkg/api/base/base.go b/pkg/api/base/base.go index 2b935130..e8288f64 100644 --- a/pkg/api/base/base.go +++ b/pkg/api/base/base.go @@ -84,3 +84,6 @@ const ( AdminUserHeader = "X-Ceems-Admin-User" ClusterIDHeader = "X-Ceems-Cluster-Id" ) + +// Username to be used for unknown users. +const UnknownUser = "unknown" diff --git a/pkg/api/cli/cli.go b/pkg/api/cli/cli.go index ac06b876..b2d28994 100644 --- a/pkg/api/cli/cli.go +++ b/pkg/api/cli/cli.go @@ -85,11 +85,12 @@ func NewCEEMSServer() (*CEEMSServer, error) { func (b *CEEMSServer) Main() error { // CLI vars var ( - configFile, webConfigFile, routePrefix, corsOrigin, maxQueryPeriod, runAsUser string - enableDebugServer, skipDeleteOldUnits, disableChecks, disableCapAwareness, dropPrivs, systemdSocket, compression bool - webListenAddresses, userHeaders []string - requestsLimit, maxProcs, compressionLevel int - externalURL *url.URL + configFile, webConfigFile, routePrefix, corsOrigin, maxQueryPeriod, runAsUser string + enableDebugServer, skipDeleteOldUnits, disableChecks bool + dropPrivs, systemdSocket, compression, disableCapAwareness bool + webListenAddresses, userHeaders []string + requestsLimit, maxProcs, compressionLevel int + externalURL *url.URL ) // Get default run as user @@ -376,9 +377,7 @@ func (b *CEEMSServer) Main() error { } // Create server instance. - apiServer, cleanup, err := ceems_http.New(serverConfig) - defer cleanup() - + apiServer, err := ceems_http.New(serverConfig) if err != nil { logger.Error("Failed to create ceems_server server", "err", err) diff --git a/pkg/api/cli/cli_test.go b/pkg/api/cli/cli_test.go index 7af2cbee..4eb78271 100644 --- a/pkg/api/cli/cli_test.go +++ b/pkg/api/cli/cli_test.go @@ -154,5 +154,6 @@ ceems_api_server: // Send INT signal and wait a second to clean up server and DB syscall.Kill(syscall.Getpid(), syscall.SIGINT) - time.Sleep(1 * time.Second) + // Seems like 1 sec can be too short in CI workflows + time.Sleep(3 * time.Second) } diff --git a/pkg/api/db/db.go b/pkg/api/db/db.go index be6172a6..7dbb9424 100644 --- a/pkg/api/db/db.go +++ b/pkg/api/db/db.go @@ -53,7 +53,7 @@ type Timezone struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (t *Timezone) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *Timezone) UnmarshalYAML(unmarshal func(any) error) error { var tmp string err := unmarshal(&tmp) @@ -77,7 +77,7 @@ type DateTime struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (t *DateTime) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *DateTime) UnmarshalYAML(unmarshal func(any) error) error { var tmp string var err error @@ -127,7 +127,7 @@ type AdminConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *AdminConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *AdminConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *c = AdminConfig{ Grafana: common.GrafanaWebConfig{ @@ -175,7 +175,7 @@ type DataConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *DataConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *DataConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config todayMidnight, _ := time.Parse("2006-01-02", time.Now().Format("2006-01-02")) *c = DataConfig{ @@ -562,8 +562,12 @@ func (s *stats) purgeExpiredUnits(ctx context.Context, tx *sql.Tx) error { defer common.TimeTrack(time.Now(), "DB cleanup", s.logger) // Purge expired units + // Check the units based on their end time rather than start time. In the + // case of long running units (like VMs and Pods), we can have these units + // running longer than retention period which will delete them from DB if + // we check based on their start time. deleteUnitsQuery := fmt.Sprintf( - "DELETE FROM %s WHERE started_at <= date('now', '-%d day')", + "DELETE FROM %s WHERE ended_at <= date('now', '-%d day')", base.UnitsDBTableName, int(s.storage.retentionPeriod.Hours()/24), ) // #nosec diff --git a/pkg/api/http/compress.go b/pkg/api/http/compress.go index e0a59625..9918b990 100644 --- a/pkg/api/http/compress.go +++ b/pkg/api/http/compress.go @@ -162,7 +162,7 @@ func (c *Compressor) SetEncoder(encoding string, fn EncoderFunc) { encoder := fn(io.Discard, c.level) if _, ok := encoder.(ioResetterWriter); ok { pool := &sync.Pool{ - New: func() interface{} { + New: func() any { return fn(io.Discard, c.level) }, } diff --git a/pkg/api/http/querier.go b/pkg/api/http/querier.go index fbfad218..63383eb5 100644 --- a/pkg/api/http/querier.go +++ b/pkg/api/http/querier.go @@ -144,7 +144,7 @@ func countRows(ctx context.Context, dbConn *sql.DB, query Query) (int, error) { defer countStmt.Close() // queryParams has to be an inteface. Do casting here - qParams := make([]interface{}, len(queryParams)) + qParams := make([]any, len(queryParams)) for i, v := range queryParams { qParams[i] = v } @@ -206,7 +206,7 @@ func Querier[T any](ctx context.Context, dbConn *sql.DB, query Query, logger *sl defer queryStmt.Close() // queryParams has to be an inteface. Do casting here - qParams := make([]interface{}, len(queryParams)) + qParams := make([]any, len(queryParams)) for i, v := range queryParams { qParams[i] = v } diff --git a/pkg/api/http/querier_test.go b/pkg/api/http/querier_test.go index c2d71ee3..13a269e2 100644 --- a/pkg/api/http/querier_test.go +++ b/pkg/api/http/querier_test.go @@ -172,8 +172,6 @@ func TestUnitsQuerier(t *testing.T) { } func TestUsageQuerier(t *testing.T) { - logger := noOpLogger - db, err := setupTestDB() require.NoError(t, err, "failed to setup test DB") defer db.Close() @@ -219,14 +217,12 @@ func TestUsageQuerier(t *testing.T) { NumUpdates: 2, }, } - usageStats, err := Querier[models.Usage](t.Context(), db, q, logger) + usageStats, err := Querier[models.Usage](t.Context(), db, q, noOpLogger) require.NoError(t, err) assert.Equal(t, expectedUsageStats, usageStats) } func TestProjectQuerier(t *testing.T) { - logger := noOpLogger - db, err := setupTestDB() require.NoError(t, err, "failed to setup test DB") defer db.Close() @@ -250,14 +246,12 @@ func TestProjectQuerier(t *testing.T) { LastUpdatedAt: "2024-07-02T14:49:39", }, } - projects, err := Querier[models.Project](t.Context(), db, q, logger) + projects, err := Querier[models.Project](t.Context(), db, q, noOpLogger) require.NoError(t, err) assert.Equal(t, expectedProjects, projects) } func TestUserQuerier(t *testing.T) { - logger := noOpLogger - db, err := setupTestDB() require.NoError(t, err, "failed to setup test DB") defer db.Close() @@ -281,14 +275,12 @@ func TestUserQuerier(t *testing.T) { LastUpdatedAt: "2024-07-02T14:49:39", }, } - users, err := Querier[models.User](t.Context(), db, q, logger) + users, err := Querier[models.User](t.Context(), db, q, noOpLogger) require.NoError(t, err) assert.Equal(t, expectedUsers, users) } func TestClusterQuerier(t *testing.T) { - logger := noOpLogger - db, err := setupTestDB() require.NoError(t, err, "failed to setup test DB") defer db.Close() @@ -307,14 +299,12 @@ func TestClusterQuerier(t *testing.T) { Manager: "slurm", }, } - clusters, err := Querier[models.Cluster](t.Context(), db, q, logger) + clusters, err := Querier[models.Cluster](t.Context(), db, q, noOpLogger) require.NoError(t, err) assert.Equal(t, expectedClusters, clusters) } func TestStatsQuerier(t *testing.T) { - logger := noOpLogger - db, err := setupTestDB() require.NoError(t, err, "failed to setup test DB") defer db.Close() @@ -334,14 +324,12 @@ func TestStatsQuerier(t *testing.T) { NumUsers: 7, }, } - stats, err := Querier[models.Stat](t.Context(), db, q, logger) + stats, err := Querier[models.Stat](t.Context(), db, q, noOpLogger) require.NoError(t, err) assert.Equal(t, expectedStats, stats) } func TestKeysQuerier(t *testing.T) { - logger := noOpLogger - db, err := setupTestDB() require.NoError(t, err, "failed to setup test DB") defer db.Close() @@ -364,7 +352,7 @@ func TestKeysQuerier(t *testing.T) { Name: "requests", }, } - keys, err := Querier[models.Key](t.Context(), db, q, logger) + keys, err := Querier[models.Key](t.Context(), db, q, noOpLogger) require.NoError(t, err) assert.Equal(t, expectedKeys, keys) } diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index 6d129121..8e135fad 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -77,7 +77,7 @@ type WebConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *WebConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *WebConfig) UnmarshalYAML(unmarshal func(any) error) error { *c = WebConfig{} type plain WebConfig @@ -185,7 +185,7 @@ func getDBStatus(dbConn *sql.DB, logger *slog.Logger) bool { } // New creates new CEEMSServer struct instance. -func New(c *Config) (*CEEMSServer, func(), error) { +func New(c *Config) (*CEEMSServer, error) { var err error router := mux.NewRouter() @@ -232,7 +232,7 @@ func New(c *Config) (*CEEMSServer, func(), error) { // Make a landing page from config landingPage, err := web.NewLandingPage(*c.Web.LandingConfig) if err != nil { - return nil, func() {}, fmt.Errorf("failed to create landing page: %w", err) + return nil, fmt.Errorf("failed to create landing page: %w", err) } // Landing page @@ -286,7 +286,7 @@ func New(c *Config) (*CEEMSServer, func(), error) { "_mutex=no&mode=ro&_busy_timeout=5000", ) if server.db, err = sql.Open(sqlite3.DriverName, dsn); err != nil { - return nil, func() {}, fmt.Errorf("failed to open DB: %w", err) + return nil, fmt.Errorf("failed to open DB: %w", err) } // Rate limit requests by RealIP @@ -298,7 +298,7 @@ func New(c *Config) (*CEEMSServer, func(), error) { // Add a middleware that verifies headers and pass them in requests amw, err := newAuthenticationMiddleware(routePrefix, c.Web.UserHeaderNames, server.db, c.Logger) if err != nil { - return nil, func() {}, fmt.Errorf("failed to create middleware: %w", err) + return nil, fmt.Errorf("failed to create middleware: %w", err) } router.Use(amw.Middleware) @@ -316,7 +316,7 @@ func New(c *Config) (*CEEMSServer, func(), error) { // starts automatic expired item deletion go server.usageCache.Start() - return server, func() {}, nil + return server, nil } // Start launches CEEMS HTTP server godoc @@ -428,6 +428,19 @@ func (s *CEEMSServer) getUser(r *http.Request) string { return r.Header.Get(base.LoggedUserHeader) } +// Get common project users. +func (s *CEEMSServer) getCommonProjectUsers(r *http.Request) []string { + var users []string + + if projects := r.URL.Query()["project"]; len(projects) > 0 { + for _, project := range projects { + users = append(users, fmt.Sprintf("%s:%s", project, base.UnknownUser)) + } + } + + return users +} + // setHeaders sets common response headers. func (s *CEEMSServer) setHeaders(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") @@ -515,10 +528,7 @@ func (s *CEEMSServer) getQueryWindow(r *http.Request, column string, running boo var fromTime, toTime time.Time // Get to and from query parameters and do checks on them - if f := q.Get("from"); f == "" { - // If from is not present in query params, use a default query window of 1 week - fromTime = time.Now().Add(-defaultQueryWindow).In(s.dbConfig.Data.Timezone.Location) - } else { + if f := q.Get("from"); f != "" { // Return error response if from is not a timestamp if ts, err := strconv.ParseInt(f, 10, 64); err != nil { s.logger.Error("Failed to parse from timestamp", "from", f, "err", err) @@ -529,10 +539,7 @@ func (s *CEEMSServer) getQueryWindow(r *http.Request, column string, running boo } } - if t := q.Get("to"); t == "" { - // Use current time as default to - toTime = time.Now().In(s.dbConfig.Data.Timezone.Location) - } else { + if t := q.Get("to"); t != "" { // Return error response if to is not a timestamp if ts, err := strconv.ParseInt(t, 10, 64); err != nil { s.logger.Error("Failed to parse to timestamp", "to", t, "err", err) @@ -543,6 +550,19 @@ func (s *CEEMSServer) getQueryWindow(r *http.Request, column string, running boo } } + switch { + // If only from is provided, add defaultQuery time to toTime + case !fromTime.IsZero() && toTime.IsZero(): + toTime = fromTime.Add(defaultQueryWindow).In(s.dbConfig.Data.Timezone.Location) + // If only toTime is provided, subtract defaultQuery time from fromTime + case fromTime.IsZero() && !toTime.IsZero(): + fromTime = toTime.Add(-defaultQueryWindow).In(s.dbConfig.Data.Timezone.Location) + // If none of them are provided, use current time + case fromTime.IsZero() && toTime.IsZero(): + toTime = time.Now().In(s.dbConfig.Data.Timezone.Location) + fromTime = toTime.Add(-defaultQueryWindow).In(s.dbConfig.Data.Timezone.Location) + } + // If difference between from and to is more than max query period, return with empty // response. This is to prevent users from making "big" requests that can "potentially" // choke server and end up in OOM errors @@ -879,8 +899,11 @@ func (s *CEEMSServer) units(w http.ResponseWriter, r *http.Request) { // Get current logged user from headers loggedUser := s.getUser(r) + // Get project specific common users and add current user to slice + queriedUsers := append(s.getCommonProjectUsers(r), loggedUser) + // Query for units and write response - s.unitsQuerier([]string{loggedUser}, w, r) + s.unitsQuerier(queriedUsers, w, r) } // verifyUnitsOwnership godoc @@ -987,7 +1010,7 @@ func (s *CEEMSServer) clustersAdmin(w http.ResponseWriter, r *http.Request) { s.setHeaders(w) // Get current user from header - loggerUser := s.getUser(r) + loggedUser := s.getUser(r) // Make query q := Query{} @@ -1001,7 +1024,7 @@ func (s *CEEMSServer) clustersAdmin(w http.ResponseWriter, r *http.Request) { // Make query and get list of cluster ids clusterIDs, err := s.queriers.cluster(r.Context(), s.db, q, s.logger) if clusterIDs == nil && err != nil { - s.logger.Error("Failed to fetch cluster IDs", "user", loggerUser, "err", err) + s.logger.Error("Failed to fetch cluster IDs", "user", loggedUser, "err", err) errorResponse[any](w, &apiError{errorInternal, err}, s.logger, nil) return @@ -1137,10 +1160,13 @@ func (s *CEEMSServer) users(w http.ResponseWriter, r *http.Request) { defer common.TimeTrack(time.Now(), "users endpoint", s.logger) // Get current user from header - loggerUser := s.getUser(r) + loggedUser := s.getUser(r) + + // Get project specific common users and add current user to slice + queriedUsers := append(s.getCommonProjectUsers(r), loggedUser) // Query for users and write response - s.usersQuerier([]string{loggerUser}, w, r) + s.usersQuerier(queriedUsers, w, r) } // usersAdmin godoc @@ -1280,10 +1306,13 @@ func (s *CEEMSServer) projects(w http.ResponseWriter, r *http.Request) { defer common.TimeTrack(time.Now(), "projects endpoint", s.logger) // Get current user from header - loggerUser := s.getUser(r) + loggedUser := s.getUser(r) + + // Get project specific common users and add current user to slice + queriedUsers := append(s.getCommonProjectUsers(r), loggedUser) // Make query and write response - s.projectsQuerier([]string{loggerUser}, w, r) + s.projectsQuerier(queriedUsers, w, r) } // projectsAdmin godoc @@ -1350,7 +1379,7 @@ func (s *CEEMSServer) aggQueryBuilder( } // Template data - data := map[string]interface{}{ + data := map[string]any{ "MetricName": metric, "TimesMetricName": "total_time_seconds", "MetricWeight": db.Weights[metric], @@ -1466,7 +1495,7 @@ func (s *CEEMSServer) currentUsage(users []string, fields []string, w http.Respo continue } - for _, p := range strings.Split(parts[1], "|") { + for p := range strings.SplitSeq(parts[1], "|") { if p == "" || slices.Contains(virtualTables, p) { continue } @@ -1679,7 +1708,10 @@ func (s *CEEMSServer) usage(w http.ResponseWriter, r *http.Request) { s.setHeaders(w) // Get current user from header - loggerUser := s.getUser(r) + loggedUser := s.getUser(r) + + // Get project specific common users and add current user to slice + queriedUsers := append(s.getCommonProjectUsers(r), loggedUser) // Get path parameter type var mode string @@ -1694,7 +1726,7 @@ func (s *CEEMSServer) usage(w http.ResponseWriter, r *http.Request) { // Get fields query parameters if any queriedFields := s.getQueriedFields(r.URL.Query(), base.UsageDBTableColNames) if len(queriedFields) == 0 { - s.logger.Error("Invalid query fields", "logged_user", loggerUser) + s.logger.Error("Invalid query fields", "logged_user", loggedUser) errorResponse[any](w, &apiError{errorBadData, errInvalidQueryField}, s.logger, nil) return @@ -1702,12 +1734,12 @@ func (s *CEEMSServer) usage(w http.ResponseWriter, r *http.Request) { // handle current usage query if mode == currentUsage { - s.currentUsage([]string{loggerUser}, queriedFields, w, r) + s.currentUsage(queriedUsers, queriedFields, w, r) } // handle global usage query if mode == globalUsage { - s.globalUsage([]string{loggerUser}, queriedFields, w, r) + s.globalUsage(queriedUsers, queriedFields, w, r) } } @@ -1779,7 +1811,7 @@ func (s *CEEMSServer) usageAdmin(w http.ResponseWriter, r *http.Request) { s.setHeaders(w) // Get current user from header - loggerUser := s.getUser(r) + loggedUser := s.getUser(r) // Get path parameter type var mode string @@ -1794,7 +1826,7 @@ func (s *CEEMSServer) usageAdmin(w http.ResponseWriter, r *http.Request) { // Get fields query parameters if any queriedFields := s.getQueriedFields(r.URL.Query(), base.UsageDBTableColNames) if len(queriedFields) == 0 { - s.logger.Error("Invalid query fields", "logged_user", loggerUser) + s.logger.Error("Invalid query fields", "logged_user", loggedUser) errorResponse[any](w, &apiError{errorBadData, errInvalidQueryField}, s.logger, nil) return diff --git a/pkg/api/http/server_test.go b/pkg/api/http/server_test.go index cb4c2c65..f960fc9d 100644 --- a/pkg/api/http/server_test.go +++ b/pkg/api/http/server_test.go @@ -77,7 +77,7 @@ var ( ) func setupServer(d string) *CEEMSServer { - server, _, _ := New( + server, _ := New( &Config{ Logger: noOpLogger, DB: db.Config{ diff --git a/pkg/api/models/types.go b/pkg/api/models/types.go index 7d31d850..f3eedf66 100644 --- a/pkg/api/models/types.go +++ b/pkg/api/models/types.go @@ -30,7 +30,7 @@ var ( // Any number will be converted into int64. // Ref: https://go.dev/play/p/89ra6QgcZba, https://husobee.github.io/golang/database/2015/06/12/scanner-valuer.html, // https://gist.github.com/jmoiron/6979540 -type Generic map[string]interface{} +type Generic map[string]any // Value implements Valuer interface. func (g Generic) Value() (driver.Value, error) { @@ -45,7 +45,7 @@ func (g Generic) Value() (driver.Value, error) { } // Scan implements Scanner interface. -func (g *Generic) Scan(v interface{}) error { +func (g *Generic) Scan(v any) error { if v == nil { return nil } @@ -64,7 +64,7 @@ func (g *Generic) Scan(v interface{}) error { // Ref: Improvable, see https://groups.google.com/g/golang-nuts/c/TDuGDJAIuVM?pli=1 // Decode into a tmp var - var tmp map[string]interface{} + var tmp map[string]any d.UseNumber() @@ -135,14 +135,14 @@ func (m MetricMap) Keys() []string { } // Values returns a slice of string representation of map values. -func (m MetricMap) Values(format string) []interface{} { +func (m MetricMap) Values(format string) []any { // Return empty string when map is nil // Useful in table generation for cacct app if len(m) == 0 { - return []interface{}{""} + return []any{""} } - s := make([]interface{}, len(m)) + s := make([]any, len(m)) // Iterate over sorted keys to get predictable order for ik, k := range m.Keys() { @@ -165,7 +165,7 @@ func (m MetricMap) Value() (driver.Value, error) { } // Scan implements Scanner interface. -func (m *MetricMap) Scan(v interface{}) error { +func (m *MetricMap) Scan(v any) error { if v == nil { return nil } @@ -210,7 +210,7 @@ func (j JSONFloat) Value() (driver.Value, error) { } // Scan implements Scanner interface. -func (j *JSONFloat) Scan(v interface{}) error { +func (j *JSONFloat) Scan(v any) error { if v == nil { return nil } @@ -266,7 +266,7 @@ func (j JSONFloat) MarshalJSON() ([]byte, error) { return json.Marshal(v) } else { // Convert to bytes by truncating to 8 decimals - return []byte(fmt.Sprintf("%.8f", v)), nil + return fmt.Appendf(nil, "%.8f", v), nil } } @@ -290,7 +290,7 @@ func (j *JSONFloat) UnmarshalJSON(v []byte) error { // List is a generic type to store slices. Only string and int slices are supported. // Any number will be converted into int64. -type List []interface{} +type List []any // Value implements Valuer interface. func (l List) Value() (driver.Value, error) { @@ -305,7 +305,7 @@ func (l List) Value() (driver.Value, error) { } // Scan implements Scanner interface. -func (l *List) Scan(v interface{}) error { +func (l *List) Scan(v any) error { if v == nil { return nil } @@ -324,7 +324,7 @@ func (l *List) Scan(v interface{}) error { // Ref: Improvable, see https://groups.google.com/g/golang-nuts/c/TDuGDJAIuVM?pli=1 // Decode into a tmp var - var tmp []interface{} + var tmp []any d.UseNumber() @@ -359,7 +359,7 @@ func (c *WebConfig) SetDirectory(dir string) { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *WebConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *WebConfig) UnmarshalYAML(unmarshal func(any) error) error { type plain WebConfig *c = WebConfig{ diff --git a/pkg/api/resource/k8s/manager.go b/pkg/api/resource/k8s/manager.go new file mode 100644 index 00000000..71edb315 --- /dev/null +++ b/pkg/api/resource/k8s/manager.go @@ -0,0 +1,498 @@ +// Package k8s implements the fetcher interface to fetch pods from k8s +// resource manager +package k8s + +import ( + "context" + "fmt" + "log/slog" + "math" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/mahendrapaipuri/ceems/internal/common" + "github.com/mahendrapaipuri/ceems/pkg/api/base" + "github.com/mahendrapaipuri/ceems/pkg/api/cli" + "github.com/mahendrapaipuri/ceems/pkg/api/models" + "github.com/mahendrapaipuri/ceems/pkg/api/resource" + "github.com/mahendrapaipuri/ceems/pkg/k8s" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + k8s_resource "k8s.io/apimachinery/pkg/api/resource" +) + +const ( + notAvailable = "N/A" +) + +// Default config values. +const defaultNSUsersListFile = "/var/run/ceems/users.yaml" + +var ( + defaultUserAnnotations = []string{"ceems.io/created-by"} + defaultGPUResourceNames = []string{ + "nvidia.com/gpu", + "amd.com/gpu", + } +) + +// k8sManager is the struct containing the configuration of a given k8s cluster. +type k8sManager struct { + logger *slog.Logger + cluster models.Cluster + client *k8s.Client + config *k8sConfig + nsUsersMap map[string][]string +} + +type k8sConfig struct { + KubeConfigFile string `yaml:"kubeconfig_file"` + NSUsersListFile string `yaml:"ns_users_list_file"` + GPUResourceNames []string `yaml:"gpu_resource_names"` + UsernameAnnotations []string `yaml:"username_annotations"` + ProjectAnnotations []string `yaml:"project_annotations"` +} + +// defaults set struct fields to default values. +func (c *k8sConfig) defaults() *k8sConfig { + // Check if config is empty + if c == nil { + return &k8sConfig{ + NSUsersListFile: defaultNSUsersListFile, + UsernameAnnotations: defaultUserAnnotations, + GPUResourceNames: defaultGPUResourceNames, + } + } else { + // When config is not nil, check for vital fields + if len(c.UsernameAnnotations) == 0 { + c.UsernameAnnotations = defaultUserAnnotations + } + + if len(c.GPUResourceNames) == 0 { + c.GPUResourceNames = defaultGPUResourceNames + } + + if c.NSUsersListFile == "" { + c.NSUsersListFile = defaultNSUsersListFile + } + + return c + } +} + +const k8sPodManager = "k8s" + +func init() { + // Register k8s manager + resource.Register(k8sPodManager, New) +} + +// New returns a new openstackManager that returns compute instances. +func New(cluster models.Cluster, logger *slog.Logger) (resource.Fetcher, error) { + // Fetch any provided from extra_config + var c k8sConfig + if err := cluster.Extra.Decode(&c); err != nil { + logger.Error("Failed to decode extra_config for k8s cluster", "id", cluster.ID, "err", err) + + return nil, err + } + + // Set defaults + config := c.defaults() + + // Make k8s client + client, err := k8s.New(config.KubeConfigFile, "", logger) + if err != nil { + logger.Error("Failed to create k8s client", "id", cluster.ID, "err", err) + + return nil, err + } + + // Get update interval from main config file + mainConfig, err := common.MakeConfig[cli.CEEMSAPIAppConfig](base.ConfigFilePath, base.ConfigFileExpandEnvVars) + if err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + // Set directory for reading files + mainConfig.SetDirectory(filepath.Dir(base.ConfigFilePath)) + + // Create a new pod informer + if err := client.NewPodInformer(time.Duration(mainConfig.Server.Data.UpdateInterval)); err != nil { + logger.Error("Failed to create k8s pod informer", "id", cluster.ID, "err", err) + + return nil, err + } + + // Start pod informer + if err := client.StartInformer(); err != nil { + logger.Error("Failed to start k8s pod informer", "id", cluster.ID, "err", err) + + return nil, err + } + + // Make k8sManager configs from clusters + k8sManager := &k8sManager{ + logger: logger, + cluster: cluster, + client: client, + config: config, + nsUsersMap: make(map[string][]string), + } + + logger.Info("Pods from k8s cluster will be fetched", "id", cluster.ID) + + return k8sManager, nil +} + +// FetchUnits fetches pods from k8s. +func (k *k8sManager) FetchUnits( + _ context.Context, + start time.Time, + end time.Time, +) ([]models.ClusterUnits, error) { + // Fetch pods from client + units := k.fetchPods(start, end) + + return []models.ClusterUnits{{Cluster: k.cluster, Units: units}}, nil +} + +// FetchUsersProjects fetches current k8s users and namespaces. +func (k *k8sManager) FetchUsersProjects( + ctx context.Context, + currentTime time.Time, +) ([]models.ClusterUsers, []models.ClusterProjects, error) { + // Fetch users and namespaces association + userModels, projectModels := k.fetchUserNSs(ctx, currentTime) + + return []models.ClusterUsers{ + {Cluster: k.cluster, Users: userModels}, + }, []models.ClusterProjects{ + {Cluster: k.cluster, Projects: projectModels}, + }, nil +} + +func (k *k8sManager) fetchPods( + start time.Time, + end time.Time, +) []models.Unit { + // Fetch all pods + pods := k.client.Pods() + + // Get current time location + loc := end.Location() + + // Transform pods into units + units := make([]models.Unit, len(pods)) + + for ipod, pod := range pods { + // Convert CreatedAt to current time location + createdAt := pod.CreationTimestamp.In(loc) + + // Always get elapsed time based on createdAt which gives + // a more stable value + elapsedTime := common.Timespan(end.Sub(createdAt)).Format("15:04:05") + + // Initialise vars + var startedAt, endedAt time.Time + + var activeTimeSeconds float64 + + var status string + + // Check Pod phase to determine if pod is running or not + switch pod.Status.Phase { + // Pod can be in Running state and still have errored containers + // The typical CrashBackLoopOff in kubectl output is a result of + // pod in Running phase but containers cannot be started. + // So, we have to check the pod phases to get "real" status of the + // pod. + // We use PodReadyToStartContainers as proxy for start time as it + // indicates that the k8s has done everything to start containers + // which launches apps. If anything is wrong with app, containers fail + // to start and based on restart polciy pod will keep trying to start + // failed containers. This should be regarded as pod "running" as containers + // are being launched (even if they fail) and thus consuming resources. + case v1.PodRunning: + // Get pod start time based on status conditions + for _, cond := range pod.Status.Conditions { + if cond.Type == v1.PodReadyToStartContainers { + if cond.Status == v1.ConditionTrue { + startedAt = cond.LastTransitionTime.In(loc) + status = fmt.Sprintf("%s/%s", v1.PodRunning, v1.PodReady) + + // If the pod has started in this update interval + // update activeTime from start till now + if startedAt.After(start) { + activeTimeSeconds = startedAt.Sub(start).Seconds() + } else { + activeTimeSeconds = end.Sub(start).Seconds() + } + } else { + status = fmt.Sprintf("%s/%s", v1.PodRunning, cond.Reason) + } + + // Break loop once the status is updated + break + } + } + // Both these status indicate pod has been terminated with or without + // exit code 1. This means we should be able to get deletion time. + case v1.PodSucceeded, v1.PodFailed: + // Get pod start time based on status conditions + for _, cond := range pod.Status.Conditions { + if cond.Type == v1.PodReadyToStartContainers && cond.Status == v1.ConditionTrue { + startedAt = cond.LastTransitionTime.In(loc) + } + } + + // Get pod deletion time + if pod.DeletionTimestamp != nil { + endedAt = pod.DeletionTimestamp.In(loc) + + // Override elapsed time for deleted pods + elapsedTime = common.Timespan(endedAt.Sub(createdAt)).Format("15:04:05") + + // If the pod has been terminated in this update interval + // update activeTime from start till termination + if endedAt.Before(end) && endedAt.After(start) { + activeTimeSeconds = endedAt.Sub(start).Seconds() + } + } + + // Set status + status = string(pod.Status.Phase) + case v1.PodPending: + status = string(v1.PodPending) + elapsedTime = notAvailable + case v1.PodUnknown: + status = string(v1.PodUnknown) + elapsedTime = notAvailable + default: + status = "Unknown" + elapsedTime = notAvailable + } + + // Check if startedAt and endedAt are valid + var startedAtString, endedAtString string + + var startedAtTS, endedAtTS int64 + + if startedAt.IsZero() { + startedAtString = notAvailable + } else { + startedAtString = startedAt.Format(base.DatetimezoneLayout) + startedAtTS = startedAt.UnixMilli() + } + + if endedAt.IsZero() { + endedAtString = notAvailable + } else { + endedAtString = endedAt.Format(base.DatetimezoneLayout) + endedAtTS = endedAt.UnixMilli() + } + + // Use namespace as fallback for project + project := pod.Namespace + + for key, value := range pod.GetObjectMeta().GetAnnotations() { + if slices.Contains(k.config.ProjectAnnotations, key) { + project = value + + break + } + } + + // Use project:unknown as fallback username + username := fmt.Sprintf("%s:%s", project, base.UnknownUser) + + for key, value := range pod.GetObjectMeta().GetAnnotations() { + if slices.Contains(k.config.UsernameAnnotations, key) { + if !strings.Contains(value, "serviceaccount") { + username = value + } + + break + } + } + + // Add username and project association to map + if !slices.Contains(k.nsUsersMap[project], username) { + k.nsUsersMap[project] = append(k.nsUsersMap[project], username) + } + + // Get resources + // PodResources have been added only in 1.32 and it is still in alpha as of 20250706. + // If pod.Spec.Resources is nil, check limits on each container and sum them up. + var cpus, cpuMem float64 + if pod.Spec.Resources != nil { + cpus = pod.Spec.Resources.Limits.Cpu().AsApproximateFloat64() + cpuMem = pod.Spec.Resources.Limits.Memory().AsApproximateFloat64() + } else { + for _, cont := range pod.Spec.Containers { + cpus += cont.Resources.Limits.Cpu().AsApproximateFloat64() + cpuMem += cont.Resources.Limits.Memory().AsApproximateFloat64() + } + } + + // Ensure cpus and cpuMem is non zero. When no resources are set on pod + // use a milliCPU as cpu resource and 1 byte as cpu memory + cpus = math.Max(cpus, 0.001) + cpuMem = math.Max(cpuMem, 1) + + // Allocation + allocation := models.Allocation{ + "vcpus": cpus, + "mem": cpuMem, + } + + // Get GPU resources + // The downside is that we are treating all types of GPUs as same + // and ideally we should have weighting factor for each GPU type + // so that we can "accurately" estimate GPU allocation time. + // Something to think for the future!! + var gpus float64 + + for _, name := range k.config.GPUResourceNames { + for _, cont := range pod.Spec.Containers { + if n := cont.Resources.Limits.Name(v1.ResourceName(name), k8s_resource.DecimalSI).AsApproximateFloat64(); n > 0 { + allocation[name] = n + gpus += n + } + } + } + + // Total time + totalTime := models.MetricMap{ + "walltime": models.JSONFloat(activeTimeSeconds), + "alloc_cputime": models.JSONFloat(cpus * activeTimeSeconds), + "alloc_cpumemtime": models.JSONFloat(cpuMem * activeTimeSeconds), + "alloc_gputime": models.JSONFloat(gpus * activeTimeSeconds), + "alloc_gpumemtime": models.JSONFloat(gpus * activeTimeSeconds), + } + + // Tags + tags := models.Tag{ + "qos": string(pod.Status.QOSClass), + } + + if len(pod.Annotations) > 0 { + tags["annotations"] = pod.Annotations + } + + if len(pod.Labels) > 0 { + tags["labels"] = pod.Labels + } + + units[ipod] = models.Unit{ + ClusterID: k.cluster.ID, + ResourceManager: k8sPodManager, + UUID: string(pod.UID), + Name: pod.Name, + User: username, + Project: project, + CreatedAt: createdAt.Format(base.DatetimezoneLayout), + CreatedAtTS: createdAt.UnixMilli(), + StartedAt: startedAtString, + StartedAtTS: startedAtTS, + EndedAt: endedAtString, + EndedAtTS: endedAtTS, + State: status, + Elapsed: elapsedTime, + TotalTime: totalTime, + Allocation: allocation, + Tags: tags, + } + } + + k.logger.Info("k8s pods fetched", "cluster_id", k.cluster.ID, "start", start, "end", end, "num_pods", len(units)) + + return units +} + +func (k *k8sManager) fetchUserNSs(ctx context.Context, current time.Time) ([]models.User, []models.Project) { + // Initialise maps + usersNSs := make(map[string][]string) + + nsUsers := make(map[string][]string) + + // Current time string + currentTime := current.Format(base.DatetimezoneLayout) + + // Check if the configmap is available to fetch users + if content, err := os.ReadFile(k.config.NSUsersListFile); err == nil { + var usersDB struct { + NSUsers map[string][]string `yaml:"users"` + } + + if err := yaml.Unmarshal(content, &usersDB); err == nil { + nsUsers = usersDB.NSUsers + + for ns, users := range usersDB.NSUsers { + for _, user := range users { + usersNSs[user] = append(usersNSs[user], ns) + } + } + } + } + + // Merge users and namespaces from RBAC + if rbacUsers, err := k.client.ListUsers(ctx, ""); err == nil { + for ns, users := range rbacUsers { + for _, user := range users { + usersNSs[user] = append(usersNSs[user], ns) + nsUsers[ns] = append(nsUsers[ns], user) + } + } + } + + // Finally merge users and namespaces from fetched pods + for ns, users := range k.nsUsersMap { + for _, user := range users { + usersNSs[user] = append(usersNSs[user], ns) + nsUsers[ns] = append(nsUsers[ns], user) + } + } + + // Remove duplicates and make user and project models + var userModels []models.User + + var projectModels []models.Project + + for ns, users := range nsUsers { + slices.Sort(users) + p := models.Project{ + Name: ns, + ClusterID: k.cluster.ID, + ResourceManager: k.cluster.Manager, + Users: models.List{slices.Compact(users)}, + LastUpdatedAt: currentTime, + } + + projectModels = append(projectModels, p) + } + + for user, nss := range usersNSs { + slices.Sort(nss) + u := models.User{ + Name: user, + ClusterID: k.cluster.ID, + ResourceManager: k.cluster.Manager, + Projects: models.List{slices.Compact(nss)}, + LastUpdatedAt: currentTime, + } + + userModels = append(userModels, u) + } + + // If we have not found any user data emit a warning + if len(userModels) == 0 && len(projectModels) == 0 { + k.logger.Warn("No users and namespaces associations found", "id", k.cluster.ID) + } + + k.logger.Info("k8s user data fetched", "cluster_id", k.cluster.ID, "num_users", len(userModels), "num_projects", len(projectModels)) + + return userModels, projectModels +} diff --git a/pkg/api/resource/k8s/manager_test.go b/pkg/api/resource/k8s/manager_test.go new file mode 100644 index 00000000..d10ca13f --- /dev/null +++ b/pkg/api/resource/k8s/manager_test.go @@ -0,0 +1,321 @@ +package k8s + +import ( + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mahendrapaipuri/ceems/pkg/api/base" + "github.com/mahendrapaipuri/ceems/pkg/api/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +var ( + start, _ = time.Parse(base.DatetimezoneLayout, "2025-07-07T12:00:00+0200") + end, _ = time.Parse(base.DatetimezoneLayout, "2025-07-07T12:15:00+0200") + current, _ = time.Parse(base.DatetimezoneLayout, "2025-07-07T12:15:00+0200") + + noOpLogger = slog.New(slog.DiscardHandler) + expectedUnits = map[string]models.Unit{ + "3a61e77f-1538-476b-8231-5af9eed40fdc": { + ClusterID: "k8s-0", + ResourceManager: "k8s", + UUID: "3a61e77f-1538-476b-8231-5af9eed40fdc", + Name: "pod31", + Project: "ns3", + Group: "", + User: "kusr3", + CreatedAt: "2025-07-07T11:16:56+0200", + StartedAt: "N/A", + EndedAt: "N/A", + CreatedAtTS: 1751879816000, + StartedAtTS: 0, + EndedAtTS: 0, + Elapsed: "N/A", + State: "Pending", + Allocation: models.Generic{ + "mem": 3.221225472e+09, "nvidia.com/gpu": 1.0, "nvidia.com/mig-4g.20gb": 2.0, "vcpus": 3.0, + }, + TotalTime: models.MetricMap{ + "alloc_cpumemtime": 0, "alloc_cputime": 0, "alloc_gpumemtime": 0, "alloc_gputime": 0, "walltime": 0, + }, + Tags: models.Generic{ + "annotations": map[string]string{"ceems.io/created-by": "kusr3"}, "qos": "Burstable", + }, + }, + "6c22124f-e9a7-450b-8915-9bf3e0716d78": { + ClusterID: "k8s-0", + ResourceManager: "k8s", + UUID: "6c22124f-e9a7-450b-8915-9bf3e0716d78", + Name: "pod11", + Project: "ns1", + Group: "", + User: "kusr1", + CreatedAt: "2025-07-07T10:56:56+0200", + StartedAt: "2025-07-07T10:56:58+0200", + EndedAt: "N/A", + CreatedAtTS: 1751878616000, + StartedAtTS: 1751878618000, + EndedAtTS: 0, + Elapsed: "01:18:04", + State: "Running/Ready", + Allocation: models.Generic{ + "mem": 1.048576e+08, "nvidia.com/gpu": 2.0, "nvidia.com/mig-1g.5gb": 1.0, "vcpus": 0.1, + }, + TotalTime: models.MetricMap{ + "alloc_cpumemtime": 9.437184e+10, "alloc_cputime": 90, "alloc_gpumemtime": 2700, "alloc_gputime": 2700, "walltime": 900, + }, + Tags: models.Generic{ + "annotations": map[string]string{"ceems.io/created-by": "kusr1"}, "qos": "BestEffort", + }, + }, + "483168fc-b347-4aa2-a9fa-9e3d220ba4c5": { + ClusterID: "k8s-0", + ResourceManager: "k8s", + UUID: "483168fc-b347-4aa2-a9fa-9e3d220ba4c5", + Name: "pod21", + Project: "ns2", + Group: "", + User: "ns2:" + base.UnknownUser, + CreatedAt: "2025-07-07T11:56:56+0200", + StartedAt: "2025-07-07T11:56:58+0200", + EndedAt: "N/A", + CreatedAtTS: 1751882216000, + StartedAtTS: 1751882218000, + EndedAtTS: 0, + Elapsed: "00:18:04", + State: "Running/Ready", + Allocation: models.Generic{ + "mem": 2.097152e+08, "nvidia.com/mig-4g.20gb": 4.0, "vcpus": 0.2, + }, + TotalTime: models.MetricMap{ + "alloc_cpumemtime": 1.8874368e+11, "alloc_cputime": 180, "alloc_gpumemtime": 3600, "alloc_gputime": 3600, "walltime": 900, + }, + Tags: models.Generic{ + "annotations": map[string]string{"ceems.io/created-by": "system:serviceaccount", "ceems.io/project": "ns2"}, "qos": "Guaranteed", + }, + }, + "6232f0c5-57fa-409a-b026-0919f60e24a6": { + ClusterID: "k8s-0", + ResourceManager: "k8s", + UUID: "6232f0c5-57fa-409a-b026-0919f60e24a6", + Name: "pod22", + Project: "ns2", + Group: "", + User: "kusr2", + CreatedAt: "2025-07-07T11:26:56+0200", + StartedAt: "2025-07-07T11:26:58+0200", + EndedAt: "2025-07-07T12:10:58+0200", + CreatedAtTS: 1751880416000, + StartedAtTS: 1751880418000, + EndedAtTS: 1751883058000, + Elapsed: "00:44:02", + State: "Succeeded", + Allocation: models.Generic{ + "mem": 3.221225472e+09, "nvidia.com/mig-1g.5gb": 1.0, "nvidia.com/mig-4g.20gb": 2.0, "vcpus": 3.0, + }, + TotalTime: models.MetricMap{ + "alloc_cpumemtime": 2.119566360576e+12, "alloc_cputime": 1974, "alloc_gpumemtime": 1974, "alloc_gputime": 1974, "walltime": 658, + }, + Tags: models.Generic{ + "annotations": map[string]string{"ceems.io/created-by": "kusr2"}, "qos": "Burstable", + }, + }, + } + expectedUsers = []models.User{ + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "rb3", Projects: models.List{[]string{"ns3"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "rb1", Projects: models.List{[]string{"ns1", "ns2"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "rb2", Projects: models.List{[]string{"ns2"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "kusr1", Projects: models.List{[]string{"ns1"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "kusr2", Projects: models.List{[]string{"ns2"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "kusr3", Projects: models.List{[]string{"ns3"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "file1", Projects: models.List{[]string{"ns1"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "file2", Projects: models.List{[]string{"ns1"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "file3", Projects: models.List{[]string{"ns3"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "ns2:" + base.UnknownUser, Projects: models.List{[]string{"ns2"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + } + expectedProjects = []models.Project{ + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "ns3", Users: models.List{[]string{"file3", "kusr3", "rb3"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "ns1", Users: models.List{[]string{"file1", "file2", "kusr1", "rb1"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + {ClusterID: "k8s-0", ResourceManager: "k8s", Name: "ns2", Users: models.List{[]string{"kusr2", "ns2:" + base.UnknownUser, "rb1", "rb2"}}, LastUpdatedAt: "2025-07-07T12:15:00+0200"}, + } +) + +func mockK8sAPIServer() *httptest.Server { + // Start test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "pods") { + if data, err := os.ReadFile("../../../collector/testdata/k8s/pods-metadata.json"); err == nil { + w.Header().Add("Content-Type", "application/json") + w.Header().Add("Content-Type", "application/vnd.kubernetes.protobuf") + w.Write(data) + + return + } + } else if strings.HasSuffix(r.URL.Path, "rolebindings") { + if data, err := os.ReadFile("../../../collector/testdata/k8s/rolebindings.json"); err == nil { + w.Header().Add("Content-Type", "application/json") + w.Header().Add("Content-Type", "application/vnd.kubernetes.protobuf") + w.Write(data) + + return + } + } else { + w.Write([]byte("KO")) + } + })) + + return server +} + +func mockConfig(tmpDir, apiURL string) (yaml.Node, string, error) { + content := ` +apiVersion: v1 +clusters: +- cluster: + server: %s + name: foo-cluster +contexts: +- context: + cluster: foo-cluster + user: foo-user + namespace: bar + name: foo-context +current-context: foo-context +kind: Config +users: +- name: foo-user + user: + token: blue-token +` + kubeconfig := filepath.Join(tmpDir, "kubeconfig") + + content = fmt.Sprintf(content, apiURL) + + err := os.WriteFile(kubeconfig, []byte(content), 0o700) //nolint:gosec + if err != nil { + return yaml.Node{}, "", err + } + + mainConfig := ` +--- +ceems_api_server: + data: + update_interval: 15m +` + mainConfigFile := filepath.Join(tmpDir, "config.yaml") + + err = os.WriteFile(mainConfigFile, []byte(mainConfig), 0o700) //nolint:gosec + if err != nil { + return yaml.Node{}, "", err + } + + usersDB := ` +--- +users: + ns1: + - file1 + - file2 + ns3: + - file3 +` + usersDBFile := filepath.Join(tmpDir, "users_db.yaml") + + err = os.WriteFile(usersDBFile, []byte(usersDB), 0o700) //nolint:gosec + if err != nil { + return yaml.Node{}, "", err + } + + config := ` +--- +kubeconfig_file: %s +ns_users_list_file: %s +gpu_resource_names: + - nvidia.com/gpu + - nvidia.com/mig-4g.20gb + - nvidia.com/mig-1g.5gb +project_annotations: + - ceems.io/project` + + cfg := fmt.Sprintf(config, kubeconfig, usersDBFile) + + var extraConfig yaml.Node + + if err := yaml.Unmarshal([]byte(cfg), &extraConfig); err == nil { + return extraConfig, mainConfigFile, nil + } else { + return yaml.Node{}, mainConfigFile, err + } +} + +func TestNew(t *testing.T) { + tmpDir := t.TempDir() + + // Start mock k8s API server + server := mockK8sAPIServer() + defer server.Close() + + // Create mock config + cfg, mainConfigFile, err := mockConfig(tmpDir, server.URL) + require.NoError(t, err) + + // mock config + cluster := models.Cluster{ + ID: "k8s-0", + Manager: "k8s", + Extra: cfg, + } + + base.ConfigFilePath = mainConfigFile + + _, err = New(cluster, noOpLogger) + require.NoError(t, err) +} + +func TestFetches(t *testing.T) { + tmpDir := t.TempDir() + + // Start mock k8s API server + server := mockK8sAPIServer() + defer server.Close() + + // Create mock config + cfg, mainConfigFile, err := mockConfig(tmpDir, server.URL) + require.NoError(t, err) + + // mock config + cluster := models.Cluster{ + ID: "k8s-0", + Manager: "k8s", + Extra: cfg, + } + + base.ConfigFilePath = mainConfigFile + + c, err := New(cluster, noOpLogger) + require.NoError(t, err) + + clusterUnits, err := c.FetchUnits(t.Context(), start, end) + require.NoError(t, err) + + gotUnits := make(map[string]models.Unit) + for _, unit := range clusterUnits[0].Units { + gotUnits[unit.UUID] = unit + } + + assert.Equal(t, expectedUnits, gotUnits) + + clusterUsers, clusterProjects, err := c.FetchUsersProjects(t.Context(), current) + require.NoError(t, err) + + assert.ElementsMatch(t, expectedUsers, clusterUsers[0].Users) + assert.ElementsMatch(t, expectedProjects, clusterProjects[0].Projects) +} diff --git a/pkg/api/resource/openstack/compute.go b/pkg/api/resource/openstack/compute.go index 08bd5c16..36d6dd7b 100644 --- a/pkg/api/resource/openstack/compute.go +++ b/pkg/api/resource/openstack/compute.go @@ -198,6 +198,11 @@ func (o *openstackManager) activeInstances(ctx context.Context, start time.Time, "extra_specs": server.Flavor.ExtraSpecs, } + // Add vGPUs to allocation when found + if vgpu > 0 { + allocation["vgpus"] = int(vgpu) + } + // Tags // IMPORTANT: Openstack user facing API does not provide any details // of the compute node or hypervisor. So, we should be fetch any of those diff --git a/pkg/api/resource/openstack/identity.go b/pkg/api/resource/openstack/identity.go index 98d4e122..33764e86 100644 --- a/pkg/api/resource/openstack/identity.go +++ b/pkg/api/resource/openstack/identity.go @@ -142,7 +142,7 @@ func (o *openstackManager) usersProjectsAssoc(ctx context.Context, current time. // Chunk by userIDs in chunks of of a given size so that we make // concurrent corresponding to chunkSize each time to get projects // of each user - userIDChunks := helper.ChunkBy[string](userIDs, chunkSize) + userIDChunks := helper.ChunkBy(userIDs, chunkSize) // Get user projects userProjects := make(map[string][]Project, len(userIDs)) diff --git a/pkg/api/resource/openstack/manager.go b/pkg/api/resource/openstack/manager.go index a68b20c0..f86b35cf 100644 --- a/pkg/api/resource/openstack/manager.go +++ b/pkg/api/resource/openstack/manager.go @@ -59,12 +59,12 @@ type openstackConfig struct { Compute string `yaml:"compute"` Identity string `yaml:"identity"` } `yaml:"api_service_endpoints"` - AuthConfig interface{} `yaml:"auth"` + AuthConfig any `yaml:"auth"` } // addAuthKey embeds AuthConfig as value under `auth` key. func (c *openstackConfig) addAuthKey() { - obj := map[string]interface{}{} + obj := map[string]any{} obj["auth"] = c.AuthConfig c.AuthConfig = obj } diff --git a/pkg/api/resource/openstack/manager_test.go b/pkg/api/resource/openstack/manager_test.go index c713d301..f0a0581e 100644 --- a/pkg/api/resource/openstack/manager_test.go +++ b/pkg/api/resource/openstack/manager_test.go @@ -48,6 +48,7 @@ var ( "name": "m10.vgpu", "swap": 0, "vcpus": 8, + "vgpus": 1, }, TotalTime: models.MetricMap{ "alloc_cpumemtime": 8.429568e+06, @@ -241,7 +242,7 @@ func mockOSIdentityAPIServer() *httptest.Server { } else if strings.HasSuffix(r.URL.Path, "tokens") { decoder := json.NewDecoder(r.Body) - var t map[string]interface{} + var t map[string]any if err := decoder.Decode(&t); err != nil { w.Write([]byte("KO")) diff --git a/pkg/api/resource/slurm/cli.go b/pkg/api/resource/slurm/cli.go index a867587d..e00ce716 100644 --- a/pkg/api/resource/slurm/cli.go +++ b/pkg/api/resource/slurm/cli.go @@ -206,7 +206,7 @@ func parseSacctCmdOutput(sacctOutput string, start time.Time, end time.Time) ([] var memString string - for _, elem := range strings.Split(components[sacctFieldMap["alloctres"]], ",") { + for elem := range strings.SplitSeq(components[sacctFieldMap["alloctres"]], ",") { // For MIG devices, it can be gres/gpu: // https://github.com/SchedMD/slurm/blob/db91ac3046b3b7b845cce4a99127db8c6f14a8e8/testsuite/expect/test39.19#L70 // Use a regex gres\/gpu:([^=]+)=(\d+) for identifying number of instances diff --git a/pkg/api/testdata/config.yml b/pkg/api/testdata/config.yml index 6711c2f5..660d120d 100644 --- a/pkg/api/testdata/config.yml +++ b/pkg/api/testdata/config.yml @@ -73,6 +73,35 @@ clusters: application_credential: id: 21dced0fd20347869b93710d2b98aae0 secret: supersecret + + - id: k8s-0 + manager: k8s + updaters: + - tsdb-0 + - tsdb-1 + extra_config: + kubeconfig_file: pkg/api/testdata/k8s/kubeconfig.yaml + ns_users_list_file: pkg/api/testdata/k8s/users.yaml + gpu_resource_names: + - nvidia.com/gpu + - nvidia.com/mig-4g.20gb + - nvidia.com/mig-1g.5gb + project_annotations: + - ceems.io/project + + - id: k8s-1 + manager: k8s + updaters: + - tsdb-0 + extra_config: + kubeconfig_file: pkg/api/testdata/k8s/kubeconfig.yaml + ns_users_list_file: pkg/api/testdata/k8s/users.yaml + gpu_resource_names: + - nvidia.com/gpu + - nvidia.com/mig-4g.20gb + - nvidia.com/mig-1g.5gb + project_annotations: + - ceems.io/project updaters: - id: tsdb-0 @@ -81,7 +110,6 @@ updaters: url: http://localhost:9090 extra_config: cutoff_duration: 0s - query_batch_size: 1 queries: # Average CPU utilisation avg_cpu_usage: @@ -123,7 +151,6 @@ updaters: url: http://localhost:9090 extra_config: cutoff_duration: 0s - query_batch_size: 1 queries: # Total IO Read stats total_io_read_stats: diff --git a/pkg/api/testdata/k8s/kubeconfig.yaml b/pkg/api/testdata/k8s/kubeconfig.yaml new file mode 100644 index 00000000..92ce64b3 --- /dev/null +++ b/pkg/api/testdata/k8s/kubeconfig.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +clusters: +- cluster: + server: http://localhost:9080 + name: foo-cluster +contexts: +- context: + cluster: foo-cluster + user: foo-user + namespace: bar + name: foo-context +current-context: foo-context +kind: Config +users: +- name: foo-user + user: + token: blue-token diff --git a/pkg/api/testdata/k8s/users.yaml b/pkg/api/testdata/k8s/users.yaml new file mode 100644 index 00000000..276a6481 --- /dev/null +++ b/pkg/api/testdata/k8s/users.yaml @@ -0,0 +1,7 @@ +--- +users: + ns1: + - file1 + - file2 + ns3: + - file3 diff --git a/pkg/api/testdata/output/e2e-test-api-server-cluster-admin-query.txt b/pkg/api/testdata/output/e2e-test-api-server-cluster-admin-query.txt index 08d4610f..0ad743c6 100644 --- a/pkg/api/testdata/output/e2e-test-api-server-cluster-admin-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-server-cluster-admin-query.txt @@ -1 +1 @@ -{"status":"success","data":[{"id":"os-0","manager":"openstack"},{"id":"os-1","manager":"openstack"},{"id":"slurm-0","manager":"slurm"},{"id":"slurm-1","manager":"slurm"}]} +{"status":"success","data":[{"id":"k8s-0","manager":"k8s"},{"id":"k8s-1","manager":"k8s"},{"id":"os-0","manager":"openstack"},{"id":"os-1","manager":"openstack"},{"id":"slurm-0","manager":"slurm"},{"id":"slurm-1","manager":"slurm"}]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-current-usage-query-k8s.txt b/pkg/api/testdata/output/e2e-test-api-server-current-usage-query-k8s.txt new file mode 100644 index 00000000..6c2a45e2 --- /dev/null +++ b/pkg/api/testdata/output/e2e-test-api-server-current-usage-query-k8s.txt @@ -0,0 +1 @@ +{"status":"success","data":[]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-global-stats-admin-query.txt b/pkg/api/testdata/output/e2e-test-api-server-global-stats-admin-query.txt index 9efcde66..2990fd8d 100644 --- a/pkg/api/testdata/output/e2e-test-api-server-global-stats-admin-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-server-global-stats-admin-query.txt @@ -1 +1 @@ -{"status":"success","data":[{"cluster_id":"os-0","resource_manager":"openstack","num_units":18,"num_inactive_units":6,"num_active_units":12,"num_projects":5,"num_users":5},{"cluster_id":"os-1","resource_manager":"openstack","num_units":18,"num_inactive_units":6,"num_active_units":12,"num_projects":5,"num_users":5},{"cluster_id":"slurm-0","resource_manager":"slurm","num_units":12,"num_inactive_units":10,"num_active_units":2,"num_projects":5,"num_users":7},{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":12,"num_inactive_units":10,"num_active_units":2,"num_projects":5,"num_users":7}]} +{"status":"success","data":[{"cluster_id":"k8s-0","resource_manager":"k8s","num_units":4,"num_inactive_units":1,"num_active_units":3,"num_projects":3,"num_users":4},{"cluster_id":"k8s-1","resource_manager":"k8s","num_units":4,"num_inactive_units":1,"num_active_units":3,"num_projects":3,"num_users":4},{"cluster_id":"os-0","resource_manager":"openstack","num_units":18,"num_inactive_units":6,"num_active_units":12,"num_projects":5,"num_users":5},{"cluster_id":"os-1","resource_manager":"openstack","num_units":18,"num_inactive_units":6,"num_active_units":12,"num_projects":5,"num_users":5},{"cluster_id":"slurm-0","resource_manager":"slurm","num_units":12,"num_inactive_units":10,"num_active_units":2,"num_projects":5,"num_users":7},{"cluster_id":"slurm-1","resource_manager":"slurm","num_units":12,"num_inactive_units":10,"num_active_units":2,"num_projects":5,"num_users":7}]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-global-usage-query-k8s.txt b/pkg/api/testdata/output/e2e-test-api-server-global-usage-query-k8s.txt new file mode 100644 index 00000000..6c2a45e2 --- /dev/null +++ b/pkg/api/testdata/output/e2e-test-api-server-global-usage-query-k8s.txt @@ -0,0 +1 @@ +{"status":"success","data":[]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-project-query-k8s.txt b/pkg/api/testdata/output/e2e-test-api-server-project-query-k8s.txt new file mode 100644 index 00000000..6c2a45e2 --- /dev/null +++ b/pkg/api/testdata/output/e2e-test-api-server-project-query-k8s.txt @@ -0,0 +1 @@ +{"status":"success","data":[]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-query-k8s.txt b/pkg/api/testdata/output/e2e-test-api-server-query-k8s.txt new file mode 100644 index 00000000..70680338 --- /dev/null +++ b/pkg/api/testdata/output/e2e-test-api-server-query-k8s.txt @@ -0,0 +1 @@ +{"status":"success","data":[{"cluster_id":"k8s-1","resource_manager":"k8s","uuid":"6232f0c5-57fa-409a-b026-0919f60e24a6","name":"pod22","project":"ns2","username":"kusr2","created_at":"2025-07-07T11:26:56+0200","started_at":"2025-07-07T11:26:58+0200","ended_at":"2025-07-07T12:10:58+0200","created_at_ts":1751880416000,"started_at_ts":1751880418000,"ended_at_ts":1751883058000,"elapsed":"00:44:02","state":"Succeeded","allocation":{"mem":3221225472,"nvidia.com/mig-1g.5gb":1,"nvidia.com/mig-4g.20gb":2,"vcpus":3},"total_time_seconds":{"alloc_cpumemtime":0,"alloc_cputime":0,"alloc_gpumemtime":0,"alloc_gputime":0,"walltime":0},"avg_cpu_usage":{"global":21.25488198},"avg_cpu_mem_usage":{"global":21.25488198},"total_cpu_energy_usage_kwh":{"total":21.25488198},"total_cpu_emissions_gms":{"emaps_total":21.25488198,"rte_total":21.25488198},"avg_gpu_usage":{"global":21.25488198},"avg_gpu_mem_usage":{"global":21.25488198},"total_gpu_energy_usage_kwh":{"total":21.25488198},"total_gpu_emissions_gms":{"emaps_total":21.25488198,"rte_total":21.25488198},"tags":{"annotations":{"ceems.io/created-by":"kusr2"},"qos":"Burstable"}}]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-running-query.txt b/pkg/api/testdata/output/e2e-test-api-server-running-query.txt index 930fac16..27f6ea29 100644 --- a/pkg/api/testdata/output/e2e-test-api-server-running-query.txt +++ b/pkg/api/testdata/output/e2e-test-api-server-running-query.txt @@ -1 +1 @@ -{"status":"success","data":[{"uuid":"1cef0381-0a5a-42e6-9e9b-3d88f84be971","started_at":"2024-10-15T17:28:50+0300","state":"ACTIVE","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True","resources:VGPU":"1"},"mem":8192,"name":"m10.vgpu","swap":0,"vcpus":8},"tags":{"az":"nova","metadata":{},"power_state":"RUNNING","reservation_id":"r-ct4kh3w1","tags":[]}},{"uuid":"1e3b7f2c-a648-41a8-b53e-4fa5bd2ae73c","started_at":"2024-10-15T16:15:42+0300","state":"ACTIVE","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True"},"mem":256,"name":"cirros256","swap":0,"vcpus":1},"tags":{"az":"nova","metadata":{},"power_state":"RUNNING","reservation_id":"r-tk530ak6","tags":[]}},{"uuid":"7fe4fa04-e4ea-4b92-84f4-45c9e78b9520","started_at":"2024-10-15T16:15:11+0300","state":"ACTIVE","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True"},"mem":192,"name":"m1.micro","swap":0,"vcpus":1},"tags":{"az":"nova","metadata":{},"power_state":"RUNNING","reservation_id":"r-ztao3fbf","tags":[]}},{"uuid":"b6eafae3-5c24-4f25-b297-5ef291d9487d","started_at":"2024-10-15T16:16:00+0300","state":"SUSPENDED","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True"},"mem":128,"name":"m1.nano","swap":0,"vcpus":1},"tags":{"az":"nova","metadata":{},"power_state":"SHUTDOWN","reservation_id":"r-ks8nrkb2","tags":[]}}]} +{"status":"success","data":[{"uuid":"1cef0381-0a5a-42e6-9e9b-3d88f84be971","started_at":"2024-10-15T17:28:50+0300","state":"ACTIVE","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True","resources:VGPU":"1"},"mem":8192,"name":"m10.vgpu","swap":0,"vcpus":8,"vgpus":1},"tags":{"az":"nova","metadata":{},"power_state":"RUNNING","reservation_id":"r-ct4kh3w1","tags":[]}},{"uuid":"1e3b7f2c-a648-41a8-b53e-4fa5bd2ae73c","started_at":"2024-10-15T16:15:42+0300","state":"ACTIVE","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True"},"mem":256,"name":"cirros256","swap":0,"vcpus":1},"tags":{"az":"nova","metadata":{},"power_state":"RUNNING","reservation_id":"r-tk530ak6","tags":[]}},{"uuid":"7fe4fa04-e4ea-4b92-84f4-45c9e78b9520","started_at":"2024-10-15T16:15:11+0300","state":"ACTIVE","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True"},"mem":192,"name":"m1.micro","swap":0,"vcpus":1},"tags":{"az":"nova","metadata":{},"power_state":"RUNNING","reservation_id":"r-ztao3fbf","tags":[]}},{"uuid":"b6eafae3-5c24-4f25-b297-5ef291d9487d","started_at":"2024-10-15T16:16:00+0300","state":"SUSPENDED","allocation":{"disk":1,"extra_specs":{"hw_rng:allowed":"True"},"mem":128,"name":"m1.nano","swap":0,"vcpus":1},"tags":{"az":"nova","metadata":{},"power_state":"SHUTDOWN","reservation_id":"r-ks8nrkb2","tags":[]}}]} diff --git a/pkg/api/testdata/output/e2e-test-api-server-user-query-k8s.txt b/pkg/api/testdata/output/e2e-test-api-server-user-query-k8s.txt new file mode 100644 index 00000000..51c1b78a --- /dev/null +++ b/pkg/api/testdata/output/e2e-test-api-server-user-query-k8s.txt @@ -0,0 +1 @@ +{"status":"success","data":[{"cluster_id":"k8s-0","resource_manager":"k8s","name":"rb1","projects":[["ns1","ns2"]]},{"cluster_id":"k8s-1","resource_manager":"k8s","name":"rb1","projects":[["ns1","ns2"]]}]} diff --git a/pkg/api/updater/tsdb/tsdb.go b/pkg/api/updater/tsdb/tsdb.go index f70ff26a..df482c20 100644 --- a/pkg/api/updater/tsdb/tsdb.go +++ b/pkg/api/updater/tsdb/tsdb.go @@ -31,6 +31,56 @@ const ( defaultQueryMinSamples = 0.5 ) +// Default queries. +var ( + defaultQueries = map[string]map[string]string{ + "avg_cpu_usage": { + "global": `avg_over_time(avg by (uuid) (uuid:ceems_cpu_usage:ratio_irate{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:])`, + }, + "avg_cpu_mem_usage": { + "global": `avg_over_time(avg by (uuid) (uuid:ceems_cpu_memory_usage:ratio{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:])`, + }, + "total_cpu_energy_usage_kwh": { + "total": `sum_over_time(sum by (uuid) (uuid:ceems_host_power_watts:pue{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 3.6e9`, + }, + "total_cpu_emissions_gms": { + "rte_total": `sum_over_time(sum by (uuid) (uuid:ceems_host_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="rte"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3`, + "emaps_total": `sum_over_time(sum by (uuid) (uuid:ceems_host_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="emaps"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3`, + "owid_total": `sum_over_time(sum by (uuid) (uuid:ceems_host_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="owid"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3`, + }, + "avg_gpu_usage": { + "global": `avg_over_time(avg by (uuid) (uuid:ceems_gpu_usage:ratio{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:])`, + }, + "avg_gpu_mem_usage": { + "global": `avg_over_time(avg by (uuid) (uuid:ceems_gpu_memory_usage:ratio{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:])`, + }, + "total_gpu_energy_usage_kwh": { + "total": `sum_over_time(sum by (uuid) (uuid:ceems_gpu_power_watts:pue{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 3.6e9`, + }, + "total_gpu_emissions_gms": { + "rte_total": `sum_over_time(sum by (uuid) (uuid:ceems_gpu_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="rte"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3`, + "emaps_total": `sum_over_time(sum by (uuid) (uuid:ceems_gpu_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="emaps"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3`, + "owid_total": `sum_over_time(sum by (uuid) (uuid:ceems_gpu_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="owid"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3`, + }, + "total_io_write_stats": { + "bytes_total": `sum by (uuid) (increase(ceems_ebpf_write_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + "requests_total": `sum by (uuid) (increase(ceems_ebpf_write_requests_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + "errors_total": `sum by (uuid) (increase(ceems_ebpf_write_errors_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + }, + "total_io_read_stats": { + "bytes_total": `sum by (uuid) (increase(ceems_ebpf_read_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + "requests_total": `sum by (uuid) (increase(ceems_ebpf_read_requests_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + "errors_total": `sum by (uuid) (increase(ceems_ebpf_read_errors_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + }, + "total_ingress_stats": { + "bytes_total": `sum by (uuid) (increase(ceems_ebpf_ingress_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + }, + "total_egress_stats": { + "bytes_total": `sum by (uuid) (increase(ceems_ebpf_egress_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf)`, + }, + } +) + // config is the container for the configuration of a given TSDB instance. type tsdbConfig struct { QueryMaxSeries int64 `yaml:"query_max_series"` @@ -41,6 +91,39 @@ type tsdbConfig struct { LabelsToDrop []string `yaml:"labels_to_drop"` } +// defaults set struct fields to default values. +func (c *tsdbConfig) defaults() *tsdbConfig { + // Check if config is empty + if c == nil { + return &tsdbConfig{ + QueryMaxSeries: defaultQueryMaxSeries, + QueryMinSamples: defaultQueryMinSamples, + Queries: defaultQueries, + } + } else { + // When config is not nil, check for vital fields + if c.QueryMaxSeries == 0 { + c.QueryMaxSeries = defaultQueryMaxSeries + } + + if c.QueryMinSamples == 0 { + c.QueryMinSamples = defaultQueryMinSamples + } + + // If no queries are provided, use default + // We should not start with defaultQueries as default value + // as unmarshal will merge the defaultQueries with user supplied + // ones in that case. This is not what we want. We should always + // prioritise the user supplied ones and only when none found, + // use default ones as last fallback solution. + if len(c.Queries) == 0 { + c.Queries = defaultQueries + } + + return c + } +} + // validate validates the config. func (c *tsdbConfig) validate() error { if c.QueryMaxSeries <= 0 { @@ -75,16 +158,16 @@ func init() { // New create a new TSDB updater. func New(instance updater.Instance, logger *slog.Logger) (updater.Updater, error) { // Make TSDB config from instances extra config - config := tsdbConfig{ - QueryMaxSeries: defaultQueryMaxSeries, - QueryMinSamples: defaultQueryMinSamples, - } - if err := instance.Extra.Decode(&config); err != nil { + var c tsdbConfig + if err := instance.Extra.Decode(&c); err != nil { logger.Error("Failed to setup TSDB updater", "id", instance.ID, "err", err) return nil, err } + // Set defaults when vital fields are not set + config := c.defaults() + // Validate config if err := config.validate(); err != nil { logger.Error("Failed to validate TSDB updater config", "instance_id", instance.ID, "err", err) @@ -107,7 +190,7 @@ func New(instance updater.Instance, logger *slog.Logger) (updater.Updater, error logger.Info("TSDB updater setup successful", "id", instance.ID) return &tsdbUpdater{ - &config, + config, tsdb, }, nil } @@ -127,7 +210,7 @@ func (t *tsdbUpdater) Update( } // Return query string from template. -func (t *tsdbUpdater) queryBuilder(name string, queryTemplate string, data map[string]interface{}) (string, error) { +func (t *tsdbUpdater) queryBuilder(name string, queryTemplate string, data map[string]any) (string, error) { tmpl := template.Must(template.New(name).Parse(queryTemplate)) builder := &strings.Builder{} @@ -169,7 +252,7 @@ func (t *tsdbUpdater) fetchAggMetrics( } // Template data - tmplData := map[string]interface{}{ + tmplData := map[string]any{ "UUIDs": strings.Join(uuids, "|"), "ScrapeInterval": settings.ScrapeInterval, "ScrapeIntervalMilli": settings.ScrapeInterval.Milliseconds(), diff --git a/pkg/api/updater/tsdb/tsdb_test.go b/pkg/api/updater/tsdb/tsdb_test.go index d29e9110..cae89e33 100644 --- a/pkg/api/updater/tsdb/tsdb_test.go +++ b/pkg/api/updater/tsdb/tsdb_test.go @@ -23,22 +23,22 @@ func mockTSDBServer() *httptest.Server { // Start test server expected := tsdb.Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "resultType": "vector", - "result": []interface{}{ - map[string]interface{}{ + "result": []any{ + map[string]any{ "metric": map[string]string{ "uuid": "1", }, - "value": []interface{}{ + "value": []any{ 12345, "1.1", }, }, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": "2", }, - "value": []interface{}{ + "value": []any{ 12345, "2.2", }, }, diff --git a/pkg/collector/ebpf.go b/pkg/collector/ebpf.go index e556e59f..5c4ec632 100644 --- a/pkg/collector/ebpf.go +++ b/pkg/collector/ebpf.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "log/slog" + "maps" "slices" "strings" "sync" @@ -223,9 +224,7 @@ func NewEbpfCollector(logger *slog.Logger, cgManager *cgroupManager) (*ebpfColle return nil, err } - for name, prog := range netColl.Programs { - bpfProgs[name] = prog - } + maps.Copy(bpfProgs, netColl.Programs) // Set configMap configMap = netColl.Maps["conf_map"] @@ -242,9 +241,7 @@ func NewEbpfCollector(logger *slog.Logger, cgManager *cgroupManager) (*ebpfColle return nil, err } - for name, prog := range vfsColl.Programs { - bpfProgs[name] = prog - } + maps.Copy(bpfProgs, vfsColl.Programs) // Set configMap if not already done if configMap == nil { @@ -905,7 +902,7 @@ func (c *ebpfCollector) discoverCgroups(cgroups []cgroup) { // aggStats returns aggregate VFS and network metrics by reading // BPF maps. This function gets executes in a security context // with relevant privileges. -func aggStats(data interface{}) error { +func aggStats(data any) error { // Assert data type var d *ebpfReadMapsCtxData diff --git a/pkg/collector/gpu.go b/pkg/collector/gpu.go index 7b9bb21c..c4d8e167 100644 --- a/pkg/collector/gpu.go +++ b/pkg/collector/gpu.go @@ -503,7 +503,7 @@ func NewGPUSMI(k8sClient *ceems_k8s.Client, logger *slog.Logger) (*GPUSMI, error smiQueryCmd = amdSMIQueryCmd } - if pods, err := k8sClient.Pods(ctx, "", opts); err == nil && len(pods) > 0 { + if pods, err := k8sClient.ListPods(ctx, "", opts); err == nil && len(pods) > 0 { vendors[iv].k8sNS = pods[0].Namespace vendors[iv].k8sPod = pods[0].Name @@ -683,7 +683,7 @@ func (g *GPUSMI) ReindexGPUs(orderMap string) { return } - for _, gpuMap := range strings.Split(orderMap, ",") { + for gpuMap := range strings.SplitSeq(orderMap, ",") { orderMap := strings.Split(gpuMap, ":") if len(orderMap) < 2 { continue @@ -977,8 +977,8 @@ func parseNvidiaSmiOutput(cmdOutput []byte) ([]Device, error) { // NVIDIA B100 and so on.. // So we try to split by "space" and hypen and attempt to test // against model names - for _, s := range strings.Split(gpu.ProductName, " ") { - for _, ss := range strings.Split(s, "-") { + for s := range strings.SplitSeq(gpu.ProductName, " ") { + for ss := range strings.SplitSeq(s, "-") { if strings.TrimSpace(ss) == model { dev.NumSMs = numSMs diff --git a/pkg/collector/helper.go b/pkg/collector/helper.go index d07eded2..becd84c9 100644 --- a/pkg/collector/helper.go +++ b/pkg/collector/helper.go @@ -28,10 +28,10 @@ var ( // Nicked from https://github.com/isauran/logger/blob/master/adapters/gokit/logger.go // Ref: https://github.com/go-kit/log/issues/35 -type logFunc func(ctx context.Context, msg string, keysAndValues ...interface{}) +type logFunc func(ctx context.Context, msg string, keysAndValues ...any) // Log retrieves the key values and formats them into slog.Logger. -func (l logFunc) Log(keyvals ...interface{}) error { +func (l logFunc) Log(keyvals ...any) error { // Extract message if present var msg string @@ -341,8 +341,8 @@ func parseRange(r string) ([]string, error) { return nil, errors.New("empty range string") } - ranges := strings.Split(r, ",") - for _, r := range ranges { + ranges := strings.SplitSeq(r, ",") + for r := range ranges { boundaries := strings.Split(r, "-") if len(boundaries) == 1 { start, err = strconv.Atoi(boundaries[0]) diff --git a/pkg/collector/helper_test.go b/pkg/collector/helper_test.go index 07916d4d..a53028bd 100644 --- a/pkg/collector/helper_test.go +++ b/pkg/collector/helper_test.go @@ -34,7 +34,7 @@ func TestGokitLogger(t *testing.T) { lvl = "info" } - kvs := []interface{}{"a", 123} + kvs := []any{"a", 123} lc := log.With(logger, kvs...) err := lc.Log("msg", "message") @@ -67,7 +67,7 @@ func TestGokitLogger(t *testing.T) { lvl = "info" } - kvs := []interface{}{"a", 123} + kvs := []any{"a", 123} lc := log.With(logger, kvs...) err := lc.Log("msg", "message") diff --git a/pkg/collector/ipmi.go b/pkg/collector/ipmi.go index bde3e0f6..6173c3ed 100644 --- a/pkg/collector/ipmi.go +++ b/pkg/collector/ipmi.go @@ -486,7 +486,7 @@ func (c *impiCollector) getPowerReadings() (map[string]float64, error) { // Parse current, min and max power readings for capmc output. func (c *impiCollector) parseCapmcOutput(stdOut []byte) (map[string]float64, error) { // Unmarshal JSON output - var data map[string]interface{} + var data map[string]any if err := json.Unmarshal(stdOut, &data); err != nil { return nil, fmt.Errorf("%s Power readings command failed", crayPowerCap) } @@ -613,7 +613,7 @@ func (c *impiCollector) doRequestInSecurityContext() (map[string]float64, error) return dataPtr.powerReadings, nil } -func dcmiPowerReading(data interface{}) error { +func dcmiPowerReading(data any) error { // Assert data var d *ipmiClientSecurityCtxData @@ -658,7 +658,7 @@ func findIPMICmd() ([]string, error) { // Get value based on regex from IPMI output. func getValue(ipmiOutput []byte, regex *regexp.Regexp) (string, error) { - for _, line := range strings.Split(string(ipmiOutput), "\n") { + for line := range strings.SplitSeq(string(ipmiOutput), "\n") { match := regex.FindStringSubmatch(line) if match == nil { continue diff --git a/pkg/collector/k8s.go b/pkg/collector/k8s.go index e45e5996..b8517084 100644 --- a/pkg/collector/k8s.go +++ b/pkg/collector/k8s.go @@ -484,7 +484,7 @@ func (c *k8sCollector) podDevices(cgroups []cgroup) { defer cancel() // Get pod devices from pod resource API - pods, err := c.k8sClient.PodDevices(ctx) + pods, err := c.k8sClient.ListPodsWithDevs(ctx) if err != nil { c.logger.Error("Failed to fetch pod resources. Pod to device mappings will not be available", "err", err) diff --git a/pkg/collector/k8s_test.go b/pkg/collector/k8s_test.go index 9a614c05..777e4689 100644 --- a/pkg/collector/k8s_test.go +++ b/pkg/collector/k8s_test.go @@ -62,7 +62,7 @@ users: ` kubeConfigFile := filepath.Join(tmpDir, "kubeconfig") - err := os.WriteFile(kubeConfigFile, []byte(fmt.Sprintf(content, server.URL)), 0o700) //nolint:gosec + err := os.WriteFile(kubeConfigFile, fmt.Appendf(nil, content, server.URL), 0o700) //nolint:gosec require.NoError(t, err) // Read pod resource response json diff --git a/pkg/collector/libvirt.go b/pkg/collector/libvirt.go index a73ef845..b2f96fb8 100644 --- a/pkg/collector/libvirt.go +++ b/pkg/collector/libvirt.go @@ -599,7 +599,7 @@ func (c *libvirtCollector) instanceCgroups() ([]cgroup, error) { } // readLibvirtXMLFile reads the libvirt's XML file inside a security context. -func readLibvirtXMLFile(data interface{}) error { +func readLibvirtXMLFile(data any) error { // Assert data var d *libvirtReadXMLSecurityCtxData diff --git a/pkg/collector/perf.go b/pkg/collector/perf.go index 587b5a6c..e85a869a 100644 --- a/pkg/collector/perf.go +++ b/pkg/collector/perf.go @@ -13,8 +13,8 @@ import ( "strings" "time" + "github.com/ceems-dev/perf-utils" "github.com/mahendrapaipuri/ceems/internal/security" - "github.com/mahendrapaipuri/perf-utils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" ) @@ -1080,7 +1080,7 @@ func (c *perfCollector) closeProfilers(activePIDs []int) error { // openProfilers is a convenience function for newProfilers receiver. This function // will be executed within a security context with necessary capabilities. -func openProfilers(data interface{}) error { +func openProfilers(data any) error { // Assert data type var d *perfProfilerSecurityCtxData @@ -1196,7 +1196,7 @@ func newCacheProfiler(pid int, profilerTypes perf.CacheProfilerType) (*perf.Cach // closeProfilers is a convenience function for closeProfilers receiver. This function // will be executed within a security context with necessary capabilities. -func closeProfilers(data interface{}) error { +func closeProfilers(data any) error { // Assert data is of perfSecurityCtxData var d *perfProfilerSecurityCtxData @@ -1288,7 +1288,7 @@ func closeCacheProfiler(profiler *perf.CacheProfiler) error { // filterPerfProcs filters the processes of each cgroup inside data pointer based on // presence of target env vars. -func filterPerfProcs(data interface{}) error { +func filterPerfProcs(data any) error { // Assert data is of perfSecurityCtxData var d *perfProcFilterSecurityCtxData diff --git a/pkg/collector/perf_test.go b/pkg/collector/perf_test.go index bb99eef1..a2895f14 100644 --- a/pkg/collector/perf_test.go +++ b/pkg/collector/perf_test.go @@ -8,9 +8,9 @@ import ( "slices" "testing" + "github.com/ceems-dev/perf-utils" "github.com/containerd/cgroups/v3" "github.com/mahendrapaipuri/ceems/internal/security" - "github.com/mahendrapaipuri/perf-utils" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" "github.com/stretchr/testify/assert" diff --git a/pkg/collector/profiling_config.go b/pkg/collector/profiling_config.go index e47489ff..92501ef0 100644 --- a/pkg/collector/profiling_config.go +++ b/pkg/collector/profiling_config.go @@ -58,7 +58,7 @@ type SessionConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *SessionConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *SessionConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *c = defaultSessionConfig @@ -96,7 +96,7 @@ type PyroscopeConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *PyroscopeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *PyroscopeConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *c = PyroscopeConfig{ URL: defaultPyroscopeURL, diff --git a/pkg/collector/profiling_test.go b/pkg/collector/profiling_test.go index 56c00be7..1e22ad97 100644 --- a/pkg/collector/profiling_test.go +++ b/pkg/collector/profiling_test.go @@ -216,7 +216,7 @@ func (m *mockSession) CollectProfiles(f pprof.CollectProfilesCallback) error { return nil } -func (m *mockSession) DebugInfo() interface{} { +func (m *mockSession) DebugInfo() any { return ebpfspy.SessionDebugInfo{ ElfCache: symtab.ElfCacheDebugInfo{ BuildIDCache: symtab.GCacheDebugInfo[elf.SymTabDebugInfo]{}, diff --git a/pkg/collector/rapl.go b/pkg/collector/rapl.go index 5a7b2e93..0c578235 100644 --- a/pkg/collector/rapl.go +++ b/pkg/collector/rapl.go @@ -342,7 +342,7 @@ func readPowerLimits(zones []sysfs.RaplZone) (map[sysfs.RaplZone]uint64, error) } // readCounters reads the RAPL counters of different zones inside a security context. -func readCounters(data interface{}) error { +func readCounters(data any) error { // Assert data var d *raplCountersSecurityCtxData diff --git a/pkg/collector/rdma.go b/pkg/collector/rdma.go index e7f2e14f..d0d25524 100644 --- a/pkg/collector/rdma.go +++ b/pkg/collector/rdma.go @@ -454,7 +454,7 @@ func (c *rdmaCollector) devMR(procCgroup map[string]string) (map[string]*mr, err // Read line by line and match dev, pid and mrlen mrs := make(map[string]*mr) - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if devMatch := devRegex.FindStringSubmatch(line); len(devMatch) > 1 { if pidMatch := pidRegex.FindStringSubmatch(line); len(pidMatch) > 1 { if uuid, ok := procCgroup[pidMatch[1]]; ok { @@ -495,7 +495,7 @@ func (c *rdmaCollector) devCQ(procCgroup map[string]string) (map[string]*cq, err // Read line by line and match dev, pid and mrlen cqs := make(map[string]*cq) - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if devMatch := devRegex.FindStringSubmatch(line); len(devMatch) > 1 { if pidMatch := pidRegex.FindStringSubmatch(line); len(pidMatch) > 1 { if uuid, ok := procCgroup[pidMatch[1]]; ok { @@ -535,7 +535,7 @@ func (c *rdmaCollector) linkQP(procCgroup map[string]string) (map[string]*qp, er // Read line by line and match dev, pid and mrlen qps := make(map[string]*qp) - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if linkMatch := linkRegex.FindStringSubmatch(line); len(linkMatch) > 1 { if pidMatch := pidRegex.FindStringSubmatch(line); len(pidMatch) > 1 { if uuid, ok := procCgroup[pidMatch[1]]; ok { @@ -565,7 +565,7 @@ func (c *rdmaCollector) linkQP(procCgroup map[string]string) (map[string]*qp, er return qps, nil } - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if linkMatch := linkRegex.FindStringSubmatch(line); len(linkMatch) > 1 { for _, hwCounter := range c.hwCounters { if pidMatch := pidRegex.FindStringSubmatch(line); len(pidMatch) > 1 { @@ -647,7 +647,7 @@ func qpMode(rdmaCmd string) (map[string]bool, error) { // Split output and get mode for each device linkMode := make(map[string]bool) - for _, line := range strings.Split(string(out), "\n") { + for line := range strings.SplitSeq(string(out), "\n") { if linkMatch := linkRegex.FindStringSubmatch(line); len(linkMatch) > 1 && strings.HasPrefix(linkMatch[1], "mlx") { if autoMatch := autoRegex.FindStringSubmatch(line); len(autoMatch) > 1 { if autoMatch[1] == "off" { diff --git a/pkg/collector/redfish.go b/pkg/collector/redfish.go index 31cb51ec..19248ae5 100644 --- a/pkg/collector/redfish.go +++ b/pkg/collector/redfish.go @@ -65,7 +65,7 @@ type redfishClientConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *redfishClientConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *redfishClientConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *c = redfishClientConfig{} c.SessionToken = true @@ -139,7 +139,7 @@ type redfishConfig struct { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *redfishConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *redfishConfig) UnmarshalYAML(unmarshal func(any) error) error { type plain redfishConfig if err := unmarshal((*plain)(c)); err != nil { diff --git a/pkg/collector/slurm.go b/pkg/collector/slurm.go index ef8ceb71..9e4e8eac 100644 --- a/pkg/collector/slurm.go +++ b/pkg/collector/slurm.go @@ -688,7 +688,7 @@ func (c *slurmCollector) jobGRESResources(uuid string, procs []procfs.Proc) *gre // readProcEnvirons reads the environment variables of processes and returns // GPU ordinals of job. This function will be executed in a security context. -func readProcEnvirons(data interface{}) error { +func readProcEnvirons(data any) error { // Assert data is of slurmSecurityCtxData var d *slurmReadProcSecurityCtxData @@ -786,7 +786,7 @@ func updateGPUAvailableShares(content string, gresType string, hostname string, var enabled bool // Split file content by new line - for _, line := range strings.Split(content, "\n") { + for line := range strings.SplitSeq(content, "\n") { // Convert to lower cases for better comparison line = strings.ToLower(line) @@ -803,7 +803,7 @@ func updateGPUAvailableShares(content string, gresType string, hostname string, var count uint64 - for _, d := range strings.Split(line, " ") { + for d := range strings.SplitSeq(line, " ") { // Check if NodeName=compute[000-010] is available // If it does, check if the current hostname is in list of // node names. @@ -828,7 +828,7 @@ func updateGPUAvailableShares(content string, gresType string, hostname string, dps := p[1][1:] // When MultipleFiles= is found, we need to split by comma - for _, dp := range strings.Split(dps, ",/") { + for dp := range strings.SplitSeq(dps, ",/") { // Trim all spaces dp = strings.TrimSpace(dp) @@ -925,9 +925,9 @@ func migInstanceIDFromDevMinor(migMinor string) (string, uint64, uint64) { var computeInstID uint64 if b, err := os.ReadFile(procFilePath("driver/nvidia-caps/mig-minors")); err == nil { - for _, line := range strings.Split(string(b), "\n") { + for line := range strings.SplitSeq(string(b), "\n") { if path := strings.Split(line, " "); len(path) >= 2 && path[1] == migMinor { - for _, p := range strings.Split(path[0], "/") { + for p := range strings.SplitSeq(path[0], "/") { if strings.Contains(p, "gpu") { gpuMinor = strings.Split(p, "gpu")[1] } diff --git a/pkg/collector/slurm_test.go b/pkg/collector/slurm_test.go index 91319c8c..92e1661f 100644 --- a/pkg/collector/slurm_test.go +++ b/pkg/collector/slurm_test.go @@ -546,7 +546,7 @@ func TestJobDevicesCaching(t *testing.T) { err = os.WriteFile( dir+"/cgroup.procs", - []byte(fmt.Sprintf("%d\n", i)), + fmt.Appendf(nil, "%d\n", i), 0o600, ) require.NoError(t, err) @@ -628,7 +628,7 @@ func TestJobDevicesCaching(t *testing.T) { err = os.WriteFile( dir+"/cgroup.procs", - []byte(fmt.Sprintf("%d\n", i)), + fmt.Appendf(nil, "%d\n", i), 0o600, ) require.NoError(t, err) diff --git a/pkg/collector/targets.go b/pkg/collector/targets.go index 7f78ab43..ba3886c8 100644 --- a/pkg/collector/targets.go +++ b/pkg/collector/targets.go @@ -230,7 +230,7 @@ func (d *targetDiscoverer) discover() ([]Target, error) { } // filterTargets filters the targets based on target env vars and return filtered targets. -func filterTargets(data interface{}) error { +func filterTargets(data any) error { // Assert data is of targetDiscovererSecurityCtxData var d *targetDiscovererSecurityCtxData diff --git a/pkg/collector/testdata/k8s/pods-metadata.json b/pkg/collector/testdata/k8s/pods-metadata.json index bb36904e..e13932a2 100644 --- a/pkg/collector/testdata/k8s/pods-metadata.json +++ b/pkg/collector/testdata/k8s/pods-metadata.json @@ -2,48 +2,199 @@ "metadata": {}, "items": [ { + "apiVersion": "v1", + "kind": "Pod", "metadata": { "name": "pod11", "namespace": "ns1", - "uid": "6c22124f-e9a7-450b-8915-9bf3e0716d78" + "uid": "6c22124f-e9a7-450b-8915-9bf3e0716d78", + "creationTimestamp": "2025-07-07T08:56:56Z", + "annotations": { + "ceems.io/created-by": "kusr1" + } }, "spec": { - "containers": null + "resources": { + "limits": { + "cpu": "100m", + "memory": "100Mi" + } + }, + "containers": [ + { + "name": "cont111", + "resources": { + "limits": { + "cpu": "50m", + "memory": "50Mi", + "nvidia.com/gpu": 2 + } + } + }, + { + "name": "cont112", + "resources": { + "limits": { + "cpu": "50m", + "memory": "50Mi", + "nvidia.com/mig-1g.5gb": 1 + } + } + } + ] }, - "status": {} + "status": { + "phase": "Running", + "startTime": "2025-07-07T08:56:58Z", + "qosClass": "BestEffort", + "conditions": [ + { + "type": "PodReadyToStartContainers", + "status": "True", + "lastTransitionTime": "2025-07-07T08:56:58Z" + } + ] + } }, { + "apiVersion": "v1", + "kind": "Pod", "metadata": { "name": "pod21", - "namespace": "ns2", - "uid": "483168fc-b347-4aa2-a9fa-9e3d220ba4c5" + "namespace": "default", + "uid": "483168fc-b347-4aa2-a9fa-9e3d220ba4c5", + "creationTimestamp": "2025-07-07T09:56:56Z", + "annotations": { + "ceems.io/created-by": "system:serviceaccount", + "ceems.io/project": "ns2" + } }, "spec": { - "containers": null + "containers": [ + { + "name": "cont211", + "resources": { + "limits": { + "cpu": "50m", + "memory": "50Mi", + "nvidia.com/mig-4g.20gb": 4 + } + } + }, + { + "name": "cont212", + "resources": { + "limits": { + "cpu": "150m", + "memory": "150Mi" + } + } + } + ] }, - "status": {} + "status": { + "phase": "Running", + "startTime": "2025-07-07T09:56:58Z", + "qosClass": "Guaranteed", + "conditions": [ + { + "type": "PodReadyToStartContainers", + "status": "True", + "lastTransitionTime": "2025-07-07T09:56:58Z" + } + ] + } }, { + "apiVersion": "v1", + "kind": "Pod", "metadata": { "name": "pod22", "namespace": "ns2", - "uid": "6232f0c5-57fa-409a-b026-0919f60e24a6" + "uid": "6232f0c5-57fa-409a-b026-0919f60e24a6", + "creationTimestamp": "2025-07-07T09:26:56Z", + "deletionTimestamp": "2025-07-07T10:10:58Z", + "annotations": { + "ceems.io/created-by": "kusr2" + } }, "spec": { - "containers": null + "containers": [ + { + "name": "cont221", + "resources": { + "limits": { + "cpu": "2", + "memory": "2Gi", + "nvidia.com/mig-4g.20gb": 2 + } + } + }, + { + "name": "cont222", + "resources": { + "limits": { + "cpu": "1", + "memory": "1Gi", + "nvidia.com/mig-1g.5gb": 1 + } + } + } + ] }, - "status": {} + "status": { + "phase": "Succeeded", + "startTime": "2025-07-07T09:26:58Z", + "qosClass": "Burstable", + "conditions": [ + { + "type": "PodReadyToStartContainers", + "status": "True", + "lastTransitionTime": "2025-07-07T09:26:58Z" + } + ] + } }, { + "apiVersion": "v1", + "kind": "Pod", "metadata": { "name": "pod31", "namespace": "ns3", - "uid": "3a61e77f-1538-476b-8231-5af9eed40fdc" + "uid": "3a61e77f-1538-476b-8231-5af9eed40fdc", + "creationTimestamp": "2025-07-07T09:16:56Z", + "annotations": { + "ceems.io/created-by": "kusr3" + } }, "spec": { - "containers": null + "containers": [ + { + "name": "cont311", + "resources": { + "limits": { + "cpu": 2, + "memory": "2Gi", + "nvidia.com/gpu": 1 + } + } + }, + { + "name": "cont312", + "resources": { + "limits": { + "cpu": 1, + "memory": "1Gi", + "nvidia.com/mig-4g.20gb": 2 + } + } + } + ] }, - "status": {} + "status": { + "phase": "Pending", + "qosClass": "Burstable" + } } ] } \ No newline at end of file diff --git a/pkg/collector/testdata/k8s/rolebindings.json b/pkg/collector/testdata/k8s/rolebindings.json new file mode 100644 index 00000000..e429379f --- /dev/null +++ b/pkg/collector/testdata/k8s/rolebindings.json @@ -0,0 +1,122 @@ +{ + "metadata": {}, + "items": [ + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "annotations": { + "rbac.authorization.kubernetes.io/autoupdate": "true" + }, + "creationTimestamp": "2025-07-06T10:08:02Z", + "labels": { + "kubernetes.io/bootstrapping": "rbac-defaults" + }, + "name": "foobind", + "namespace": "ns1", + "resourceVersion": "202", + "uid": "c467a6cb-30cb-4a94-8c8b-f4fd4d9ddb8a" + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": "foorole" + }, + "subjects": [ + { + "kind": "User", + "name": "rb1" + }, + { + "kind": "ServiceAccount", + "name": "default", + "namespace": "ns1" + } + ] + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "annotations": { + "rbac.authorization.kubernetes.io/autoupdate": "true" + }, + "creationTimestamp": "2025-07-06T10:08:02Z", + "labels": { + "kubernetes.io/bootstrapping": "rbac-defaults" + }, + "name": "foobind", + "namespace": "ns2", + "resourceVersion": "202", + "uid": "7a6cbc46-30cb-4a94-8c8b-f4fd4d9ddb8a" + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": "foorole" + }, + "subjects": [ + { + "kind": "User", + "name": "rb1" + } + ] + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "annotations": { + "rbac.authorization.kubernetes.io/autoupdate": "true" + }, + "creationTimestamp": "2025-07-06T10:08:02Z", + "labels": { + "kubernetes.io/bootstrapping": "rbac-defaults" + }, + "name": "foopbind", + "namespace": "ns2", + "resourceVersion": "202", + "uid": "76252885-5928-483d-bc34-2e76d5edd717" + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": "fooprole" + }, + "subjects": [ + { + "kind": "User", + "name": "rb2" + } + ] + }, + { + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "annotations": { + "rbac.authorization.kubernetes.io/autoupdate": "true" + }, + "creationTimestamp": "2025-07-06T10:08:02Z", + "labels": { + "kubernetes.io/bootstrapping": "rbac-defaults" + }, + "name": "foopbind", + "namespace": "ns3", + "resourceVersion": "202", + "uid": "28857625-5928-483d-bc34-2e76d5edd717" + }, + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": "fooprole" + }, + "subjects": [ + { + "kind": "User", + "name": "rb3" + } + ] + } + ] + } diff --git a/pkg/emissions/data/carbon-intensity-owid.csv b/pkg/emissions/data/carbon-intensity-owid.csv index cc1cdd59..f8ecd862 100644 --- a/pkg/emissions/data/carbon-intensity-owid.csv +++ b/pkg/emissions/data/carbon-intensity-owid.csv @@ -1,83 +1,86 @@ Entity,Code,Year,Carbon intensity of electricity - gCO2/kWh -ASEAN (Ember),,2000,569.55707 -ASEAN (Ember),,2001,567.6423 -ASEAN (Ember),,2002,570.2117 -ASEAN (Ember),,2003,574.69885 -ASEAN (Ember),,2004,581.4201 -ASEAN (Ember),,2005,583.16754 -ASEAN (Ember),,2006,582.43024 -ASEAN (Ember),,2007,586.1944 -ASEAN (Ember),,2008,576.5174 -ASEAN (Ember),,2009,577.68304 -ASEAN (Ember),,2010,582.38684 -ASEAN (Ember),,2011,576.0771 -ASEAN (Ember),,2012,574.9611 -ASEAN (Ember),,2013,571.3796 -ASEAN (Ember),,2014,577.73224 -ASEAN (Ember),,2015,591.55695 -ASEAN (Ember),,2016,588.45886 -ASEAN (Ember),,2017,568.1904 -ASEAN (Ember),,2018,573.7153 -ASEAN (Ember),,2019,593.81854 -ASEAN (Ember),,2020,584.5369 -ASEAN (Ember),,2021,570.68945 -ASEAN (Ember),,2022,554.4995 -ASEAN (Ember),,2023,571.2219 -Afghanistan,AFG,2000,249.99998 +ASEAN (Ember),,2000,572.85864 +ASEAN (Ember),,2001,570.6064 +ASEAN (Ember),,2002,573.028 +ASEAN (Ember),,2003,577.3394 +ASEAN (Ember),,2004,583.91736 +ASEAN (Ember),,2005,585.559 +ASEAN (Ember),,2006,584.7877 +ASEAN (Ember),,2007,588.49133 +ASEAN (Ember),,2008,578.75305 +ASEAN (Ember),,2009,579.845 +ASEAN (Ember),,2010,584.52734 +ASEAN (Ember),,2011,578.1553 +ASEAN (Ember),,2012,576.8671 +ASEAN (Ember),,2013,573.1555 +ASEAN (Ember),,2014,579.469 +ASEAN (Ember),,2015,595.1134 +ASEAN (Ember),,2016,591.48975 +ASEAN (Ember),,2017,572.0665 +ASEAN (Ember),,2018,577.8942 +ASEAN (Ember),,2019,597.71484 +ASEAN (Ember),,2020,589.474 +ASEAN (Ember),,2021,571.4162 +ASEAN (Ember),,2022,554.26764 +ASEAN (Ember),,2023,570.0053 +ASEAN (Ember),,2024,569.94794 +Afghanistan,AFG,2000,250 Afghanistan,AFG,2001,217.39131 Afghanistan,AFG,2002,169.01408 Afghanistan,AFG,2003,241.75824 -Afghanistan,AFG,2004,227.84811 +Afghanistan,AFG,2004,227.8481 Afghanistan,AFG,2005,219.5122 -Afghanistan,AFG,2006,255.55556 +Afghanistan,AFG,2006,255.55557 Afghanistan,AFG,2007,227.72278 -Afghanistan,AFG,2008,256.41025 +Afghanistan,AFG,2008,256.41028 Afghanistan,AFG,2009,123.595505 Afghanistan,AFG,2010,147.72726 Afghanistan,AFG,2011,194.8052 Afghanistan,AFG,2012,147.72726 Afghanistan,AFG,2013,180.18018 Afghanistan,AFG,2014,129.31035 -Afghanistan,AFG,2015,118.644066 -Afghanistan,AFG,2016,115.702484 +Afghanistan,AFG,2015,118.64407 +Afghanistan,AFG,2016,115.70248 Afghanistan,AFG,2017,126.98412 Afghanistan,AFG,2018,153.84616 -Afghanistan,AFG,2019,149.5327 -Afghanistan,AFG,2020,125 -Afghanistan,AFG,2021,132.53012 -Afghanistan,AFG,2022,132.53012 -Africa,,2000,621.15717 -Africa,,2001,619.7057 -Africa,,2002,617.1339 -Africa,,2003,625.6825 +Afghanistan,AFG,2019,116.438354 +Afghanistan,AFG,2020,93.220345 +Afghanistan,AFG,2021,113.20755 +Afghanistan,AFG,2022,125 +Afghanistan,AFG,2023,123.711334 +Africa,,2000,621.1572 +Africa,,2001,619.70575 +Africa,,2002,617.13385 +Africa,,2003,625.68256 Africa,,2004,623.8753 Africa,,2005,625.1627 Africa,,2006,627.5392 Africa,,2007,627.22253 -Africa,,2008,622.01416 -Africa,,2009,614.8451 -Africa,,2010,611.52014 +Africa,,2008,622.0142 +Africa,,2009,614.84515 +Africa,,2010,611.5201 Africa,,2011,610.9393 -Africa,,2012,609.45667 -Africa,,2013,604.23883 +Africa,,2012,609.4566 +Africa,,2013,604.2389 Africa,,2014,598.4014 Africa,,2015,597.2398 Africa,,2016,594.6043 -Africa,,2017,588.3122 -Africa,,2018,583.57056 -Africa,,2019,574.8308 -Africa,,2020,564.4458 -Africa,,2021,559.33386 -Africa,,2022,554.15314 +Africa,,2017,588.33 +Africa,,2018,583.82495 +Africa,,2019,574.4222 +Africa,,2020,564.8341 +Africa,,2021,558.31793 +Africa,,2022,550.2502 +Africa,,2023,545.90845 Africa (Ember),,2000,621.0892 -Africa (Ember),,2001,619.7711 +Africa (Ember),,2001,619.77106 Africa (Ember),,2002,617.1748 Africa (Ember),,2003,625.70215 Africa (Ember),,2004,624.0234 -Africa (Ember),,2005,625.2162 +Africa (Ember),,2005,625.21625 Africa (Ember),,2006,627.52216 Africa (Ember),,2007,627.2061 -Africa (Ember),,2008,621.99805 +Africa (Ember),,2008,621.9981 Africa (Ember),,2009,614.86115 Africa (Ember),,2010,611.55725 Africa (Ember),,2011,610.903 @@ -86,61 +89,64 @@ Africa (Ember),,2013,604.3274 Africa (Ember),,2014,598.4354 Africa (Ember),,2015,597.3489 Africa (Ember),,2016,594.6372 -Africa (Ember),,2017,588.36926 -Africa (Ember),,2018,583.6616 -Africa (Ember),,2019,574.90936 -Africa (Ember),,2020,564.5617 -Africa (Ember),,2021,559.42334 -Africa (Ember),,2022,554.2751 -Africa (Ember),,2023,545.145 +Africa (Ember),,2017,588.38696 +Africa (Ember),,2018,583.91547 +Africa (Ember),,2019,574.5344 +Africa (Ember),,2020,564.92615 +Africa (Ember),,2021,558.42816 +Africa (Ember),,2022,550.2832 +Africa (Ember),,2023,541.35474 +Africa (Ember),,2024,542.41693 Albania,ALB,2000,48.625793 -Albania,ALB,2001,51.49051 -Albania,ALB,2002,61.994606 -Albania,ALB,2003,36.32887 +Albania,ALB,2001,51.490513 +Albania,ALB,2002,61.99461 +Albania,ALB,2003,36.328873 Albania,ALB,2004,39.639637 Albania,ALB,2005,33.39518 Albania,ALB,2006,35.714287 Albania,ALB,2007,42.40283 -Albania,ALB,2008,23.936172 -Albania,ALB,2009,23.076921 +Albania,ALB,2008,23.93617 +Albania,ALB,2009,23.076923 Albania,ALB,2010,23.778072 Albania,ALB,2011,33.412888 -Albania,ALB,2012,23.305084 -Albania,ALB,2013,24.42529 -Albania,ALB,2014,23.305084 +Albania,ALB,2012,23.305086 +Albania,ALB,2013,24.425287 +Albania,ALB,2014,23.305086 Albania,ALB,2015,23.7691 Albania,ALB,2016,23.136248 -Albania,ALB,2017,24.282558 -Albania,ALB,2018,23.391811 -Albania,ALB,2019,23.076921 -Albania,ALB,2020,24.482107 -Albania,ALB,2021,23.437498 -Albania,ALB,2022,24.285715 +Albania,ALB,2017,24.28256 +Albania,ALB,2018,23.391813 +Albania,ALB,2019,23.076923 +Albania,ALB,2020,24.48211 +Albania,ALB,2021,23.489933 +Albania,ALB,2022,24.390245 +Albania,ALB,2023,24.417313 Algeria,DZA,2000,638.58264 Algeria,DZA,2001,638.2419 Algeria,DZA,2002,637.9747 Algeria,DZA,2003,634.0886 Algeria,DZA,2004,634.56 -Algeria,DZA,2005,629.3129 +Algeria,DZA,2005,629.31287 Algeria,DZA,2006,635.8217 Algeria,DZA,2007,635.92365 Algeria,DZA,2008,635.34674 Algeria,DZA,2009,634.5455 Algeria,DZA,2010,636.99976 Algeria,DZA,2011,633.3463 -Algeria,DZA,2012,632.7526 +Algeria,DZA,2012,632.75256 Algeria,DZA,2013,635.9993 Algeria,DZA,2014,637.0002 -Algeria,DZA,2015,637.4201 -Algeria,DZA,2016,636.40204 +Algeria,DZA,2015,637.42017 +Algeria,DZA,2016,636.402 Algeria,DZA,2017,634.1528 -Algeria,DZA,2018,633.03345 -Algeria,DZA,2019,632.9983 -Algeria,DZA,2020,633.6784 -Algeria,DZA,2021,634.52313 -Algeria,DZA,2022,634.611 -American Samoa,ASM,2000,647.0588 -American Samoa,ASM,2001,647.0588 +Algeria,DZA,2018,632.7036 +Algeria,DZA,2019,632.76556 +Algeria,DZA,2020,632.60675 +Algeria,DZA,2021,633.528 +Algeria,DZA,2022,633.6503 +Algeria,DZA,2023,633.64484 +American Samoa,ASM,2000,647.05884 +American Samoa,ASM,2001,647.05884 American Samoa,ASM,2002,666.6666 American Samoa,ASM,2003,684.2105 American Samoa,ASM,2004,684.2105 @@ -155,14 +161,15 @@ American Samoa,ASM,2012,687.5 American Samoa,ASM,2013,666.6666 American Samoa,ASM,2014,666.6666 American Samoa,ASM,2015,687.5 -American Samoa,ASM,2016,647.0588 +American Samoa,ASM,2016,647.05884 American Samoa,ASM,2017,687.5 American Samoa,ASM,2018,687.5 American Samoa,ASM,2019,687.5 -American Samoa,ASM,2020,647.0588 -American Samoa,ASM,2021,647.0588 +American Samoa,ASM,2020,647.05884 +American Samoa,ASM,2021,647.05884 American Samoa,ASM,2022,611.1111 -Angola,AGO,2000,256.94446 +American Samoa,ASM,2023,647.05884 +Angola,AGO,2000,256.94443 Angola,AGO,2001,262.19513 Angola,AGO,2002,250 Angola,AGO,2003,261.30652 @@ -172,98 +179,103 @@ Angola,AGO,2006,117.44966 Angola,AGO,2007,169.75308 Angola,AGO,2008,162.96297 Angola,AGO,2009,236.25255 -Angola,AGO,2010,227.10622 +Angola,AGO,2010,227.10623 Angola,AGO,2011,208.48056 -Angola,AGO,2012,207.25388 +Angola,AGO,2012,207.25389 Angola,AGO,2013,289.57056 Angola,AGO,2014,324.26776 -Angola,AGO,2015,312.88342 +Angola,AGO,2015,312.88345 Angola,AGO,2016,286.88525 Angola,AGO,2017,205.3654 Angola,AGO,2018,154.4901 Angola,AGO,2019,186.47282 -Angola,AGO,2020,171.32661 +Angola,AGO,2020,171.3266 Angola,AGO,2021,172.7003 -Angola,AGO,2022,174.73436 +Angola,AGO,2022,172.15483 +Angola,AGO,2023,167.22408 Antigua and Barbuda,ATG,2000,666.6666 -Antigua and Barbuda,ATG,2001,647.0588 +Antigua and Barbuda,ATG,2001,647.05884 Antigua and Barbuda,ATG,2002,684.2105 Antigua and Barbuda,ATG,2003,666.6667 Antigua and Barbuda,ATG,2004,636.36365 Antigua and Barbuda,ATG,2005,666.6667 Antigua and Barbuda,ATG,2006,653.8462 -Antigua and Barbuda,ATG,2007,655.17236 +Antigua and Barbuda,ATG,2007,655.1724 Antigua and Barbuda,ATG,2008,666.6666 Antigua and Barbuda,ATG,2009,666.6666 Antigua and Barbuda,ATG,2010,666.6666 Antigua and Barbuda,ATG,2011,666.6666 Antigua and Barbuda,ATG,2012,666.6666 Antigua and Barbuda,ATG,2013,666.6666 -Antigua and Barbuda,ATG,2014,647.0588 +Antigua and Barbuda,ATG,2014,647.05884 Antigua and Barbuda,ATG,2015,628.5714 -Antigua and Barbuda,ATG,2016,647.0588 +Antigua and Barbuda,ATG,2016,647.05884 Antigua and Barbuda,ATG,2017,628.5714 -Antigua and Barbuda,ATG,2018,638.8889 -Antigua and Barbuda,ATG,2019,638.8889 +Antigua and Barbuda,ATG,2018,638.88885 +Antigua and Barbuda,ATG,2019,638.88885 Antigua and Barbuda,ATG,2020,628.5714 Antigua and Barbuda,ATG,2021,628.5714 Antigua and Barbuda,ATG,2022,611.1111 +Antigua and Barbuda,ATG,2023,611.1111 Argentina,ARG,2000,355.4731 -Argentina,ARG,2001,302.08566 +Argentina,ARG,2001,302.08563 Argentina,ARG,2002,294.83392 -Argentina,ARG,2003,316.38968 +Argentina,ARG,2003,316.3897 Argentina,ARG,2004,357.67004 Argentina,ARG,2005,357.16312 Argentina,ARG,2006,349.9074 Argentina,ARG,2007,388.03735 -Argentina,ARG,2008,400.62338 +Argentina,ARG,2008,400.6234 Argentina,ARG,2009,382.35776 -Argentina,ARG,2010,401.5873 +Argentina,ARG,2010,401.58728 Argentina,ARG,2011,419.87796 -Argentina,ARG,2012,432.14233 -Argentina,ARG,2013,422.23022 +Argentina,ARG,2012,432.1423 +Argentina,ARG,2013,422.2302 Argentina,ARG,2014,424.13745 Argentina,ARG,2015,426.55756 Argentina,ARG,2016,431.9201 -Argentina,ARG,2017,425.38284 +Argentina,ARG,2017,425.38287 Argentina,ARG,2018,414.29352 -Argentina,ARG,2019,397.6206 +Argentina,ARG,2019,397.62057 Argentina,ARG,2020,394.7041 Argentina,ARG,2021,406.98663 -Argentina,ARG,2022,394.61646 -Argentina,ARG,2023,354.10287 +Argentina,ARG,2022,392.87186 +Argentina,ARG,2023,353.95755 +Argentina,ARG,2024,358.94824 Armenia,ARM,2000,280.2768 Armenia,ARM,2001,290.94077 Armenia,ARM,2002,184.55745 Armenia,ARM,2003,179.24527 -Armenia,ARM,2004,172.71156 +Armenia,ARM,2004,172.71158 Armenia,ARM,2005,184.21053 -Armenia,ARM,2006,161.12083 +Armenia,ARM,2006,161.12085 Armenia,ARM,2007,162.25749 Armenia,ARM,2008,193.22034 Armenia,ARM,2009,134.19118 -Armenia,ARM,2010,146.73045 +Armenia,ARM,2010,146.73047 Armenia,ARM,2011,203.8835 Armenia,ARM,2012,263.4271 Armenia,ARM,2013,256.3418 Armenia,ARM,2014,264.2762 Armenia,ARM,2015,225.16556 -Armenia,ARM,2016,222.53523 +Armenia,ARM,2016,222.53522 Armenia,ARM,2017,231.0757 Armenia,ARM,2018,268.7747 Armenia,ARM,2019,248.33112 Armenia,ARM,2020,251.3089 Armenia,ARM,2021,265.54404 -Armenia,ARM,2022,264.53815 -Aruba,ABW,2000,653.8461 +Armenia,ARM,2022,240.8907 +Armenia,ARM,2023,262.42776 +Armenia,ARM,2024,244.3439 +Aruba,ABW,2000,653.8462 Aruba,ABW,2001,654.3209 Aruba,ABW,2002,658.5366 -Aruba,ABW,2003,654.7619 +Aruba,ABW,2003,654.76196 Aruba,ABW,2004,655.1724 -Aruba,ABW,2005,659.3407 -Aruba,ABW,2006,659.3407 +Aruba,ABW,2005,659.34064 +Aruba,ABW,2006,659.34064 Aruba,ABW,2007,659.57446 -Aruba,ABW,2008,659.3407 +Aruba,ABW,2008,659.34064 Aruba,ABW,2009,642.1053 Aruba,ABW,2010,585.1064 Aruba,ABW,2011,580.6452 @@ -271,214 +283,212 @@ Aruba,ABW,2012,554.3478 Aruba,ABW,2013,553.19147 Aruba,ABW,2014,542.55316 Aruba,ABW,2015,536.08246 -Aruba,ABW,2016,563.8297 +Aruba,ABW,2016,563.8298 Aruba,ABW,2017,567.0103 -Aruba,ABW,2018,552.0833 -Aruba,ABW,2019,556.70105 -Aruba,ABW,2020,556.70105 -Aruba,ABW,2021,561.2245 -Aruba,ABW,2022,561.2245 -Asia,,2000,628.63104 -Asia,,2001,629.984 -Asia,,2002,637.18256 -Asia,,2003,656.25244 -Asia,,2004,648.98334 -Asia,,2005,656.2372 -Asia,,2006,660.6378 -Asia,,2007,671.25586 -Asia,,2008,662.21173 -Asia,,2009,667.2178 -Asia,,2010,660.04535 -Asia,,2011,675.67065 -Asia,,2012,672.84784 -Asia,,2013,672.3121 -Asia,,2014,663.58636 -Asia,,2015,649.91724 -Asia,,2016,642.0913 -Asia,,2017,636.987 -Asia,,2018,632.9895 -Asia,,2019,618.8107 -Asia,,2020,608.4806 -Asia,,2021,603.64233 -Asia,,2022,594.40784 -Asia (Ember),,2000,624.6731 -Asia (Ember),,2001,624.8755 -Asia (Ember),,2002,635.091 -Asia (Ember),,2003,657.54755 -Asia (Ember),,2004,650.61163 -Asia (Ember),,2005,658.2588 -Asia (Ember),,2006,663.95465 -Asia (Ember),,2007,674.8576 -Asia (Ember),,2008,663.1215 -Asia (Ember),,2009,668.9626 -Asia (Ember),,2010,662.2951 -Asia (Ember),,2011,679.9281 -Asia (Ember),,2012,677.2085 -Asia (Ember),,2013,677.46326 -Asia (Ember),,2014,665.8842 -Asia (Ember),,2015,651.7133 -Asia (Ember),,2016,642.8444 -Asia (Ember),,2017,636.78143 -Asia (Ember),,2018,631.89374 -Asia (Ember),,2019,618.764 -Asia (Ember),,2020,606.95 -Asia (Ember),,2021,601.44055 -Asia (Ember),,2022,591.19104 -Asia (Ember),,2023,589.90985 +Aruba,ABW,2018,546.3917 +Aruba,ABW,2019,545.4546 +Aruba,ABW,2020,545.4546 +Aruba,ABW,2021,550 +Aruba,ABW,2022,550 +Asia,,2000,629.7672 +Asia,,2001,631.0938 +Asia,,2002,638.1716 +Asia,,2003,657.21545 +Asia,,2004,649.8792 +Asia,,2005,657.19763 +Asia,,2006,661.58295 +Asia,,2007,672.2628 +Asia,,2008,663.23334 +Asia,,2009,668.1679 +Asia,,2010,660.82806 +Asia,,2011,676.50055 +Asia,,2012,673.58276 +Asia,,2013,672.9437 +Asia,,2014,664.34174 +Asia,,2015,650.66583 +Asia,,2016,642.3821 +Asia,,2017,638.02716 +Asia,,2018,634.2912 +Asia,,2019,619.91785 +Asia,,2020,609.47455 +Asia,,2021,604.41455 +Asia,,2022,594.30774 +Asia,,2023,593.83777 +Asia (Ember),,2000,624.9652 +Asia (Ember),,2001,625.1408 +Asia (Ember),,2002,635.34625 +Asia (Ember),,2003,657.7839 +Asia (Ember),,2004,650.83014 +Asia (Ember),,2005,658.4668 +Asia (Ember),,2006,664.15015 +Asia (Ember),,2007,675.04767 +Asia (Ember),,2008,663.30676 +Asia (Ember),,2009,669.1408 +Asia (Ember),,2010,662.4746 +Asia (Ember),,2011,680.0745 +Asia (Ember),,2012,677.3134 +Asia (Ember),,2013,677.5442 +Asia (Ember),,2014,666.0522 +Asia (Ember),,2015,651.8957 +Asia (Ember),,2016,642.545 +Asia (Ember),,2017,637.2289 +Asia (Ember),,2018,632.51685 +Asia (Ember),,2019,619.1246 +Asia (Ember),,2020,607.1584 +Asia (Ember),,2021,601.2024 +Asia (Ember),,2022,590.86273 +Asia (Ember),,2023,590.39417 +Asia (Ember),,2024,573.0421 Australia,AUS,2000,805.977 Australia,AUS,2001,803.1303 Australia,AUS,2002,794.66644 -Australia,AUS,2003,792.1665 -Australia,AUS,2004,792.8281 -Australia,AUS,2005,791.80023 +Australia,AUS,2003,792.16644 +Australia,AUS,2004,792.8282 +Australia,AUS,2005,791.8003 Australia,AUS,2006,790.4954 -Australia,AUS,2007,791.1509 +Australia,AUS,2007,791.15094 Australia,AUS,2008,791.507 Australia,AUS,2009,785.1141 Australia,AUS,2010,775.6704 Australia,AUS,2011,749.4147 Australia,AUS,2012,746.988 -Australia,AUS,2013,726.1581 +Australia,AUS,2013,726.158 Australia,AUS,2014,734.18134 Australia,AUS,2015,729.71375 Australia,AUS,2016,713.51117 Australia,AUS,2017,718.0755 -Australia,AUS,2018,688.421 +Australia,AUS,2018,688.42096 Australia,AUS,2019,670.1015 -Australia,AUS,2020,642.33466 +Australia,AUS,2020,642.3157 Australia,AUS,2021,605.90765 -Australia,AUS,2022,570.35284 -Australia,AUS,2023,548.69226 -Austria,AUT,1990,249.84778 -Austria,AUT,1991,262.7592 -Austria,AUT,1992,201.64362 -Austria,AUT,1993,183.58994 -Austria,AUT,1994,196.15013 -Austria,AUT,1995,207.26613 -Austria,AUT,1996,233.30214 -Austria,AUT,1997,233.31535 -Austria,AUT,1998,210.31886 -Austria,AUT,1999,204.62157 +Australia,AUS,2022,577.86163 +Australia,AUS,2023,556.296 +Australia,AUS,2024,551.58984 Austria,AUT,2000,203.387 Austria,AUT,2001,229.0915 -Austria,AUT,2002,226.12569 +Austria,AUT,2002,226.12567 Austria,AUT,2003,282.38745 Austria,AUT,2004,259.33713 -Austria,AUT,2005,254.00029 -Austria,AUT,2006,253.01398 +Austria,AUT,2005,254.0003 +Austria,AUT,2006,253.01399 Austria,AUT,2007,232.69138 Austria,AUT,2008,220.41006 Austria,AUT,2009,195.89186 Austria,AUT,2010,226.77676 Austria,AUT,2011,238.5852 -Austria,AUT,2012,185.87035 +Austria,AUT,2012,185.87036 Austria,AUT,2013,173.35397 Austria,AUT,2014,150.4468 -Austria,AUT,2015,172.06215 +Austria,AUT,2015,172.06216 Austria,AUT,2016,154.61691 Austria,AUT,2017,162.9155 Austria,AUT,2018,157.87846 Austria,AUT,2019,152.77974 Austria,AUT,2020,133.40092 -Austria,AUT,2021,136.0228 -Austria,AUT,2022,141.78989 -Austria,AUT,2023,110.81243 +Austria,AUT,2021,135.15941 +Austria,AUT,2022,141.0634 +Austria,AUT,2023,111.61346 +Austria,AUT,2024,102.62287 Azerbaijan,AZE,2000,617.773 Azerbaijan,AZE,2001,648.7342 Azerbaijan,AZE,2002,623.6616 -Azerbaijan,AZE,2003,615.2399 -Azerbaijan,AZE,2004,611.87854 -Azerbaijan,AZE,2005,609.8949 +Azerbaijan,AZE,2003,615.23987 +Azerbaijan,AZE,2004,611.8785 +Azerbaijan,AZE,2005,609.89496 Azerbaijan,AZE,2006,630.76294 Azerbaijan,AZE,2007,631.07245 -Azerbaijan,AZE,2008,639.6855 +Azerbaijan,AZE,2008,639.6854 Azerbaijan,AZE,2009,629.1777 -Azerbaijan,AZE,2010,588.64484 +Azerbaijan,AZE,2010,588.6449 Azerbaijan,AZE,2011,623.58167 -Azerbaijan,AZE,2012,659.5559 +Azerbaijan,AZE,2012,659.55597 Azerbaijan,AZE,2013,669.2473 -Azerbaijan,AZE,2014,677.2229 -Azerbaijan,AZE,2015,663.8177 +Azerbaijan,AZE,2014,677.22296 +Azerbaijan,AZE,2015,663.8176 Azerbaijan,AZE,2016,651.5091 Azerbaijan,AZE,2017,658.95953 Azerbaijan,AZE,2018,661.75885 Azerbaijan,AZE,2019,667.8227 Azerbaijan,AZE,2020,680 -Azerbaijan,AZE,2021,676.1426 -Azerbaijan,AZE,2022,671.38904 -Bahamas,BHS,2000,658.82355 -Bahamas,BHS,2001,655.3672 +Azerbaijan,AZE,2021,676.1425 +Azerbaijan,AZE,2022,671.3891 +Azerbaijan,AZE,2023,669.9897 +Azerbaijan,AZE,2024,633.0714 +Bahamas,BHS,2000,658.8235 +Bahamas,BHS,2001,655.36725 Bahamas,BHS,2002,656.08466 Bahamas,BHS,2003,660 Bahamas,BHS,2004,656.5656 Bahamas,BHS,2005,660.2871 Bahamas,BHS,2006,655.6604 -Bahamas,BHS,2007,657.77783 -Bahamas,BHS,2008,657.77783 -Bahamas,BHS,2009,658.8784 +Bahamas,BHS,2007,657.7778 +Bahamas,BHS,2008,657.7778 +Bahamas,BHS,2009,658.8785 Bahamas,BHS,2010,656.71643 Bahamas,BHS,2011,658.5366 -Bahamas,BHS,2012,657.8947 +Bahamas,BHS,2012,657.8948 Bahamas,BHS,2013,659.57446 Bahamas,BHS,2014,656.25 Bahamas,BHS,2015,660 Bahamas,BHS,2016,659.89844 -Bahamas,BHS,2017,658.2914 +Bahamas,BHS,2017,658.29144 Bahamas,BHS,2018,660 Bahamas,BHS,2019,656.71643 -Bahamas,BHS,2020,658.41583 -Bahamas,BHS,2021,660.0986 -Bahamas,BHS,2022,660.0986 +Bahamas,BHS,2020,655.1724 +Bahamas,BHS,2021,656.8628 +Bahamas,BHS,2022,653.65857 Bahrain,BHR,2000,904.7619 Bahrain,BHR,2001,904.92474 Bahrain,BHR,2002,904.98035 Bahrain,BHR,2003,904.4586 -Bahrain,BHR,2004,904.6439 +Bahrain,BHR,2004,904.644 Bahrain,BHR,2005,904.49146 Bahrain,BHR,2006,898.7642 Bahrain,BHR,2007,901.4733 Bahrain,BHR,2008,904.90796 -Bahrain,BHR,2009,904.69855 +Bahrain,BHR,2009,904.6986 Bahrain,BHR,2010,904.7019 -Bahrain,BHR,2011,904.6836 +Bahrain,BHR,2011,904.68365 Bahrain,BHR,2012,904.72345 Bahrain,BHR,2013,904.67004 Bahrain,BHR,2014,904.2553 Bahrain,BHR,2015,904.52795 -Bahrain,BHR,2016,904.2778 +Bahrain,BHR,2016,904.2777 Bahrain,BHR,2017,904.5502 -Bahrain,BHR,2018,904.5685 -Bahrain,BHR,2019,904.5318 -Bahrain,BHR,2020,904.4052 -Bahrain,BHR,2021,904.3683 -Bahrain,BHR,2022,904.6145 +Bahrain,BHR,2018,904.56854 +Bahrain,BHR,2019,904.25854 +Bahrain,BHR,2020,904.1342 +Bahrain,BHR,2021,903.56824 +Bahrain,BHR,2022,902.7466 +Bahrain,BHR,2023,902.40533 Bangladesh,BGD,2000,645.54584 Bangladesh,BGD,2001,647.2958 Bangladesh,BGD,2002,647.4934 Bangladesh,BGD,2003,648.2312 Bangladesh,BGD,2004,653.0044 -Bangladesh,BGD,2005,654.67365 +Bangladesh,BGD,2005,654.6736 Bangladesh,BGD,2006,655.8356 Bangladesh,BGD,2007,673.12036 Bangladesh,BGD,2008,672.5095 -Bangladesh,BGD,2009,681.9769 +Bangladesh,BGD,2009,681.97687 Bangladesh,BGD,2010,675.2325 Bangladesh,BGD,2011,672.5463 Bangladesh,BGD,2012,673.79675 -Bangladesh,BGD,2013,673.7907 +Bangladesh,BGD,2013,673.7908 Bangladesh,BGD,2014,676.53876 Bangladesh,BGD,2015,675.8306 -Bangladesh,BGD,2016,672.6011 +Bangladesh,BGD,2016,672.601 Bangladesh,BGD,2017,672.15796 -Bangladesh,BGD,2018,672.58655 -Bangladesh,BGD,2019,675.43036 -Bangladesh,BGD,2020,677.21375 -Bangladesh,BGD,2021,678.8626 -Bangladesh,BGD,2022,678.1058 -Bangladesh,BGD,2023,691.4112 -Barbados,BRB,2000,658.22784 -Barbados,BRB,2001,650.6024 -Barbados,BRB,2002,662.7907 +Bangladesh,BGD,2018,673.0259 +Bangladesh,BGD,2019,675.24384 +Bangladesh,BGD,2020,678.9141 +Bangladesh,BGD,2021,679.03815 +Bangladesh,BGD,2022,676.4792 +Bangladesh,BGD,2023,683.11975 +Bangladesh,BGD,2024,694.62494 +Barbados,BRB,2000,658.2278 +Barbados,BRB,2001,650.6025 +Barbados,BRB,2002,662.79065 Barbados,BRB,2003,666.6666 Barbados,BRB,2004,662.9213 Barbados,BRB,2005,656.5656 @@ -489,17 +499,18 @@ Barbados,BRB,2009,663.55133 Barbados,BRB,2010,660.1942 Barbados,BRB,2011,663.46155 Barbados,BRB,2012,654.86725 -Barbados,BRB,2013,653.46533 +Barbados,BRB,2013,653.4654 Barbados,BRB,2014,650.4855 Barbados,BRB,2015,644.23083 Barbados,BRB,2016,638.0953 Barbados,BRB,2017,634.6154 Barbados,BRB,2018,634.6154 Barbados,BRB,2019,632.0755 -Barbados,BRB,2020,613.2075 +Barbados,BRB,2020,613.2076 Barbados,BRB,2021,603.7736 Barbados,BRB,2022,605.5046 -Belarus,BLR,2000,518.0076 +Barbados,BRB,2023,600 +Belarus,BLR,2000,518.0077 Belarus,BLR,2001,516.55365 Belarus,BLR,2002,515.4951 Belarus,BLR,2003,514.08185 @@ -510,52 +521,45 @@ Belarus,BLR,2007,508.95386 Belarus,BLR,2008,512.12555 Belarus,BLR,2009,534.07965 Belarus,BLR,2010,511.17477 -Belarus,BLR,2011,509.1643 +Belarus,BLR,2011,509.16434 Belarus,BLR,2012,510.22397 Belarus,BLR,2013,505.87112 -Belarus,BLR,2014,507.1963 +Belarus,BLR,2014,507.19632 Belarus,BLR,2015,506.89352 Belarus,BLR,2016,506.10663 -Belarus,BLR,2017,499.8551 +Belarus,BLR,2017,499.85513 Belarus,BLR,2018,501.15414 -Belarus,BLR,2019,498.76425 +Belarus,BLR,2019,498.76422 Belarus,BLR,2020,498.7026 -Belarus,BLR,2021,432.72018 -Belarus,BLR,2022,441.74 -Belgium,BEL,1990,312.98904 -Belgium,BEL,1991,310.71777 -Belgium,BEL,1992,301.7362 -Belgium,BEL,1993,310.78766 -Belgium,BEL,1994,325.76395 -Belgium,BEL,1995,319.91296 -Belgium,BEL,1996,304.09903 -Belgium,BEL,1997,272.85568 -Belgium,BEL,1998,291.75296 -Belgium,BEL,1999,245.65192 +Belarus,BLR,2021,433.21213 +Belarus,BLR,2022,439.21463 +Belarus,BLR,2023,364.38733 +Belarus,BLR,2024,313.6173 Belgium,BEL,2000,269.59778 Belgium,BEL,2001,252.82971 Belgium,BEL,2002,249.81467 -Belgium,BEL,2003,249.13245 +Belgium,BEL,2003,249.13246 Belgium,BEL,2004,247.61789 -Belgium,BEL,2005,241.05238 +Belgium,BEL,2005,241.0524 Belgium,BEL,2006,231.58148 -Belgium,BEL,2007,224.2829 -Belgium,BEL,2008,218.27592 -Belgium,BEL,2009,216.91174 -Belgium,BEL,2010,212.88031 -Belgium,BEL,2011,192.98442 -Belgium,BEL,2012,202.67453 -Belgium,BEL,2013,184.2587 +Belgium,BEL,2007,224.28291 +Belgium,BEL,2008,218.27594 +Belgium,BEL,2009,216.91176 +Belgium,BEL,2010,212.83456 +Belgium,BEL,2011,192.89795 +Belgium,BEL,2012,202.67451 +Belgium,BEL,2013,184.25871 Belgium,BEL,2014,189.88765 -Belgium,BEL,2015,215.40259 -Belgium,BEL,2016,158.7998 -Belgium,BEL,2017,159.83127 -Belgium,BEL,2018,187.33083 -Belgium,BEL,2019,156.86696 -Belgium,BEL,2020,166.85545 -Belgium,BEL,2021,131.90462 -Belgium,BEL,2022,139.79181 -Belgium,BEL,2023,138.1068 +Belgium,BEL,2015,215.54874 +Belgium,BEL,2016,158.70004 +Belgium,BEL,2017,159.47974 +Belgium,BEL,2018,186.95004 +Belgium,BEL,2019,156.54332 +Belgium,BEL,2020,166.61003 +Belgium,BEL,2021,131.80402 +Belgium,BEL,2022,139.4042 +Belgium,BEL,2023,134.91486 +Belgium,BEL,2024,117.57641 Belize,BLZ,2000,230.76924 Belize,BLZ,2001,230.76924 Belize,BLZ,2002,214.2857 @@ -565,20 +569,21 @@ Belize,BLZ,2005,375 Belize,BLZ,2006,95.2381 Belize,BLZ,2007,136.36363 Belize,BLZ,2008,47.61905 -Belize,BLZ,2009,124.99999 -Belize,BLZ,2010,142.85713 +Belize,BLZ,2009,125 +Belize,BLZ,2010,142.85715 Belize,BLZ,2011,90.90909 Belize,BLZ,2012,131.57895 -Belize,BLZ,2013,108.10811 +Belize,BLZ,2013,108.1081 Belize,BLZ,2014,138.88889 Belize,BLZ,2015,138.88889 -Belize,BLZ,2016,105.26316 +Belize,BLZ,2016,105.26315 Belize,BLZ,2017,100 Belize,BLZ,2018,146.34146 -Belize,BLZ,2019,173.07695 -Belize,BLZ,2020,214.2857 -Belize,BLZ,2021,214.2857 -Belize,BLZ,2022,225.80646 +Belize,BLZ,2019,173.07693 +Belize,BLZ,2020,159.09091 +Belize,BLZ,2021,166.66666 +Belize,BLZ,2022,170.21277 +Belize,BLZ,2023,155.55556 Benin,BEN,2000,625 Benin,BEN,2001,666.6667 Benin,BEN,2002,666.6667 @@ -588,264 +593,265 @@ Benin,BEN,2005,636.36365 Benin,BEN,2006,666.6666 Benin,BEN,2007,636.36365 Benin,BEN,2008,652.17395 -Benin,BEN,2009,692.3078 +Benin,BEN,2009,692.30774 Benin,BEN,2010,636.36365 Benin,BEN,2011,700 Benin,BEN,2012,600 Benin,BEN,2013,615.38464 -Benin,BEN,2014,647.0588 +Benin,BEN,2014,647.05884 Benin,BEN,2015,625 -Benin,BEN,2016,666.66675 +Benin,BEN,2016,666.6667 Benin,BEN,2017,625 Benin,BEN,2018,631.5789 -Benin,BEN,2019,611.11115 +Benin,BEN,2019,611.1111 Benin,BEN,2020,595.95953 Benin,BEN,2021,584.0708 -Benin,BEN,2022,584.0708 -Bermuda,BMU,2000,650 +Benin,BEN,2022,595.95953 +Benin,BEN,2023,590 +Bermuda,BMU,2000,649.99994 Bermuda,BMU,2001,639.34424 Bermuda,BMU,2002,640.625 -Bermuda,BMU,2003,651.5152 -Bermuda,BMU,2004,651.5152 +Bermuda,BMU,2003,651.51514 +Bermuda,BMU,2004,651.51514 Bermuda,BMU,2005,652.1739 Bermuda,BMU,2006,647.8873 -Bermuda,BMU,2007,643.83563 +Bermuda,BMU,2007,643.8356 Bermuda,BMU,2008,652.7778 Bermuda,BMU,2009,648.6486 Bermuda,BMU,2010,648.6486 -Bermuda,BMU,2011,643.83563 +Bermuda,BMU,2011,643.8356 Bermuda,BMU,2012,642.8571 -Bermuda,BMU,2013,647.0588 -Bermuda,BMU,2014,651.5152 -Bermuda,BMU,2015,641.7911 -Bermuda,BMU,2016,651.5152 -Bermuda,BMU,2017,641.7911 +Bermuda,BMU,2013,647.05884 +Bermuda,BMU,2014,651.51514 +Bermuda,BMU,2015,641.791 +Bermuda,BMU,2016,651.51514 +Bermuda,BMU,2017,641.791 Bermuda,BMU,2018,640.625 Bermuda,BMU,2019,639.34424 -Bermuda,BMU,2020,644.06775 -Bermuda,BMU,2021,644.06775 +Bermuda,BMU,2020,644.0678 +Bermuda,BMU,2021,644.0678 Bermuda,BMU,2022,650.79364 +Bermuda,BMU,2023,640.625 Bhutan,BTN,2000,22.346369 -Bhutan,BTN,2001,26.041666 +Bhutan,BTN,2001,26.041668 Bhutan,BTN,2002,22.93578 Bhutan,BTN,2003,24 Bhutan,BTN,2004,25.751074 -Bhutan,BTN,2005,22.900763 -Bhutan,BTN,2006,24.608503 +Bhutan,BTN,2005,22.900764 +Bhutan,BTN,2006,24.608501 Bhutan,BTN,2007,23.112482 Bhutan,BTN,2008,23.222061 -Bhutan,BTN,2009,23.088024 +Bhutan,BTN,2009,23.088022 Bhutan,BTN,2010,23.415977 Bhutan,BTN,2011,24.390245 Bhutan,BTN,2012,23.738873 Bhutan,BTN,2013,23.904383 Bhutan,BTN,2014,23.776224 -Bhutan,BTN,2015,23.285902 -Bhutan,BTN,2016,24.111673 +Bhutan,BTN,2015,23.2859 +Bhutan,BTN,2016,24.111675 Bhutan,BTN,2017,23.529413 Bhutan,BTN,2018,23.222061 -Bhutan,BTN,2019,23.463686 -Bhutan,BTN,2020,23.463686 -Bhutan,BTN,2021,23.333334 -Bhutan,BTN,2022,23.333334 -Bolivia,BOL,2000,368.5567 -Bolivia,BOL,2001,345.95963 +Bhutan,BTN,2019,24.277456 +Bhutan,BTN,2020,23.746702 +Bhutan,BTN,2021,24.029575 +Bhutan,BTN,2022,24.14113 +Bhutan,BTN,2023,24.19355 +Bolivia,BOL,2000,368.55667 +Bolivia,BOL,2001,345.9596 Bolivia,BOL,2002,350.96155 Bolivia,BOL,2003,404.15704 -Bolivia,BOL,2004,390.72842 +Bolivia,BOL,2004,390.72845 Bolivia,BOL,2005,443.76282 Bolivia,BOL,2006,423.99997 Bolivia,BOL,2007,424.63232 Bolivia,BOL,2008,451.01352 -Bolivia,BOL,2009,465.37842 -Bolivia,BOL,2010,494.73685 -Bolivia,BOL,2011,493.65298 +Bolivia,BOL,2009,465.37845 +Bolivia,BOL,2010,494.73682 +Bolivia,BOL,2011,493.653 Bolivia,BOL,2012,504.72336 Bolivia,BOL,2013,495.51855 Bolivia,BOL,2014,530.1205 -Bolivia,BOL,2015,526.6667 -Bolivia,BOL,2016,588.17206 +Bolivia,BOL,2015,526.6666 +Bolivia,BOL,2016,588.172 Bolivia,BOL,2017,554.7445 -Bolivia,BOL,2018,521.8274 -Bolivia,BOL,2019,474.92624 -Bolivia,BOL,2020,482.9317 -Bolivia,BOL,2021,470.91934 -Bolivia,BOL,2022,489.13986 -Bolivia,BOL,2023,531.68726 +Bolivia,BOL,2018,523.9044 +Bolivia,BOL,2019,478.3028 +Bolivia,BOL,2020,483.93375 +Bolivia,BOL,2021,467.3413 +Bolivia,BOL,2022,485.86118 +Bolivia,BOL,2023,489.11224 +Bolivia,BOL,2024,468.0162 Bosnia and Herzegovina,BIH,2000,517.7373 Bosnia and Herzegovina,BIH,2001,517.40356 Bosnia and Herzegovina,BIH,2002,515.29193 Bosnia and Herzegovina,BIH,2003,601.59717 Bosnia and Herzegovina,BIH,2004,534.17126 Bosnia and Herzegovina,BIH,2005,526.98413 -Bosnia and Herzegovina,BIH,2006,562.82916 +Bosnia and Herzegovina,BIH,2006,562.8292 Bosnia and Herzegovina,BIH,2007,659.5925 -Bosnia and Herzegovina,BIH,2008,671.4383 +Bosnia and Herzegovina,BIH,2008,671.43835 Bosnia and Herzegovina,BIH,2009,600.7752 Bosnia and Herzegovina,BIH,2010,531.01 Bosnia and Herzegovina,BIH,2011,708.6928 Bosnia and Herzegovina,BIH,2012,700.2172 -Bosnia and Herzegovina,BIH,2013,584.9275 +Bosnia and Herzegovina,BIH,2013,584.92755 Bosnia and Herzegovina,BIH,2014,653.55255 -Bosnia and Herzegovina,BIH,2015,664.02924 +Bosnia and Herzegovina,BIH,2015,664.0292 Bosnia and Herzegovina,BIH,2016,685.48846 Bosnia and Herzegovina,BIH,2017,766.0098 Bosnia and Herzegovina,BIH,2018,660.7237 Bosnia and Herzegovina,BIH,2019,637.19867 Bosnia and Herzegovina,BIH,2020,711.1375 Bosnia and Herzegovina,BIH,2021,615.9696 -Bosnia and Herzegovina,BIH,2022,670.09064 -Bosnia and Herzegovina,BIH,2023,600.00006 +Bosnia and Herzegovina,BIH,2022,683.1804 +Bosnia and Herzegovina,BIH,2023,612.3295 +Bosnia and Herzegovina,BIH,2024,638.04486 Botswana,BWA,2000,850.87726 Botswana,BWA,2001,849.5575 Botswana,BWA,2002,852.17395 -Botswana,BWA,2003,855.7693 -Botswana,BWA,2004,845.36084 -Botswana,BWA,2005,858.4906 +Botswana,BWA,2003,855.7692 +Botswana,BWA,2004,845.3608 +Botswana,BWA,2005,858.49066 Botswana,BWA,2006,858.5859 Botswana,BWA,2007,855.4217 Botswana,BWA,2008,857.1429 -Botswana,BWA,2009,854.8386 +Botswana,BWA,2009,854.8387 Botswana,BWA,2010,849.05664 Botswana,BWA,2011,854.1667 Botswana,BWA,2012,800 Botswana,BWA,2013,826.0869 Botswana,BWA,2014,852.8139 -Botswana,BWA,2015,848.4848 -Botswana,BWA,2016,858.73596 -Botswana,BWA,2017,857.6159 +Botswana,BWA,2015,848.48486 +Botswana,BWA,2016,858.736 +Botswana,BWA,2017,857.61584 Botswana,BWA,2018,855.30554 -Botswana,BWA,2019,852.5897 +Botswana,BWA,2019,852.58966 Botswana,BWA,2020,848.62384 -Botswana,BWA,2021,847.38947 -Botswana,BWA,2022,847.9087 -Brazil,BRA,2000,89.44488 +Botswana,BWA,2021,847.3895 +Botswana,BWA,2022,847.6563 +Botswana,BWA,2023,849.4209 +Brazil,BRA,2000,89.44487 Brazil,BRA,2001,105.73734 Brazil,BRA,2002,97.30418 Brazil,BRA,2003,90.67208 -Brazil,BRA,2004,101.783264 +Brazil,BRA,2004,101.78327 Brazil,BRA,2005,98.891014 Brazil,BRA,2006,95.95899 Brazil,BRA,2007,90.91318 Brazil,BRA,2008,113.62753 Brazil,BRA,2009,87.37156 Brazil,BRA,2010,114.15278 -Brazil,BRA,2011,99.76304 +Brazil,BRA,2011,99.76305 Brazil,BRA,2012,128.3439 Brazil,BRA,2013,167.8433 Brazil,BRA,2014,192.9388 Brazil,BRA,2015,188.6377 -Brazil,BRA,2016,149.64844 -Brazil,BRA,2017,155.87456 +Brazil,BRA,2016,149.64845 +Brazil,BRA,2017,155.87457 Brazil,BRA,2018,136.9494 Brazil,BRA,2019,141.93233 -Brazil,BRA,2020,133.35773 -Brazil,BRA,2021,168.05109 -Brazil,BRA,2022,105.51264 -Brazil,BRA,2023,98.34824 +Brazil,BRA,2020,133.35771 +Brazil,BRA,2021,168.0511 +Brazil,BRA,2022,103.32709 +Brazil,BRA,2023,96.39573 +Brazil,BRA,2024,103.212296 British Virgin Islands,VGB,2000,636.36365 British Virgin Islands,VGB,2001,666.6667 British Virgin Islands,VGB,2002,642.8572 British Virgin Islands,VGB,2003,642.8572 British Virgin Islands,VGB,2004,666.6666 British Virgin Islands,VGB,2005,687.5 -British Virgin Islands,VGB,2006,647.0588 +British Virgin Islands,VGB,2006,647.05884 British Virgin Islands,VGB,2007,666.6666 British Virgin Islands,VGB,2008,666.6666 British Virgin Islands,VGB,2009,684.2105 -British Virgin Islands,VGB,2010,650 +British Virgin Islands,VGB,2010,649.99994 British Virgin Islands,VGB,2011,684.2105 -British Virgin Islands,VGB,2012,650 -British Virgin Islands,VGB,2013,650 -British Virgin Islands,VGB,2014,650 +British Virgin Islands,VGB,2012,649.99994 +British Virgin Islands,VGB,2013,649.99994 +British Virgin Islands,VGB,2014,649.99994 British Virgin Islands,VGB,2015,666.6667 British Virgin Islands,VGB,2016,636.36365 -British Virgin Islands,VGB,2017,647.0588 +British Virgin Islands,VGB,2017,647.05884 British Virgin Islands,VGB,2018,687.5 British Virgin Islands,VGB,2019,687.5 British Virgin Islands,VGB,2020,687.5 -British Virgin Islands,VGB,2021,647.0588 -British Virgin Islands,VGB,2022,647.0588 +British Virgin Islands,VGB,2021,647.05884 +British Virgin Islands,VGB,2022,647.05884 +British Virgin Islands,VGB,2023,647.05884 Brunei,BRN,2000,901.5748 -Brunei,BRN,2001,903.1007 -Brunei,BRN,2002,904.05896 +Brunei,BRN,2001,903.10077 +Brunei,BRN,2002,904.059 Brunei,BRN,2003,903.125 Brunei,BRN,2004,901.8405 Brunei,BRN,2005,901.8405 Brunei,BRN,2006,903.03033 -Brunei,BRN,2007,902.6548 +Brunei,BRN,2007,902.65485 Brunei,BRN,2008,903.5087 Brunei,BRN,2009,903.31494 Brunei,BRN,2010,902.3747 -Brunei,BRN,2011,903.48517 +Brunei,BRN,2011,903.4852 Brunei,BRN,2012,903.30786 -Brunei,BRN,2013,904.5455 -Brunei,BRN,2014,904.4445 +Brunei,BRN,2013,904.5454 +Brunei,BRN,2014,904.44446 Brunei,BRN,2015,902.381 Brunei,BRN,2016,901.63934 -Brunei,BRN,2017,901.4424 +Brunei,BRN,2017,901.4423 Brunei,BRN,2018,902.32556 Brunei,BRN,2019,896.55176 Brunei,BRN,2020,895.4704 Brunei,BRN,2021,894.7369 -Brunei,BRN,2022,893.913 -Bulgaria,BGR,1990,558.7112 -Bulgaria,BGR,1991,565.24945 -Bulgaria,BGR,1992,561.51245 -Bulgaria,BGR,1993,519.556 -Bulgaria,BGR,1994,507.109 -Bulgaria,BGR,1995,477.7911 -Bulgaria,BGR,1996,459.4658 -Bulgaria,BGR,1997,488.4922 -Bulgaria,BGR,1998,485.75574 -Bulgaria,BGR,1999,476.14026 -Bulgaria,BGR,2000,458.24094 -Bulgaria,BGR,2001,484.34628 +Brunei,BRN,2022,892.8572 +Brunei,BRN,2023,892.6654 +Bulgaria,BGR,2000,458.24097 +Bulgaria,BGR,2001,484.34625 Bulgaria,BGR,2002,444.62924 Bulgaria,BGR,2003,492.90445 -Bulgaria,BGR,2004,493.71375 -Bulgaria,BGR,2005,453.8619 +Bulgaria,BGR,2004,493.7137 +Bulgaria,BGR,2005,453.86194 Bulgaria,BGR,2006,451.8046 Bulgaria,BGR,2007,558.6175 Bulgaria,BGR,2008,550.9103 Bulgaria,BGR,2009,528.4803 -Bulgaria,BGR,2010,521.0102 +Bulgaria,BGR,2010,521.01025 Bulgaria,BGR,2011,577.6622 Bulgaria,BGR,2012,525.3982 Bulgaria,BGR,2013,487.44766 -Bulgaria,BGR,2014,486.99915 -Bulgaria,BGR,2015,494.55963 -Bulgaria,BGR,2016,472.04132 +Bulgaria,BGR,2014,486.99918 +Bulgaria,BGR,2015,494.55966 +Bulgaria,BGR,2016,472.04135 Bulgaria,BGR,2017,502.89917 Bulgaria,BGR,2018,442.7005 -Bulgaria,BGR,2019,438.70084 -Bulgaria,BGR,2020,383.9063 +Bulgaria,BGR,2019,438.7008 +Bulgaria,BGR,2020,383.90634 Bulgaria,BGR,2021,415.29013 -Bulgaria,BGR,2022,476.90164 -Bulgaria,BGR,2023,335.3338 -Burkina Faso,BFA,2000,487.17944 +Bulgaria,BGR,2022,476.33255 +Bulgaria,BGR,2023,338.0703 +Bulgaria,BGR,2024,264.20685 +Burkina Faso,BFA,2000,487.1795 Burkina Faso,BFA,2001,555.55554 Burkina Faso,BFA,2002,555.55554 -Burkina Faso,BFA,2003,511.1111 +Burkina Faso,BFA,2003,511.11115 Burkina Faso,BFA,2004,510.63828 -Burkina Faso,BFA,2005,538.4616 +Burkina Faso,BFA,2005,538.46155 Burkina Faso,BFA,2006,563.63635 Burkina Faso,BFA,2007,540.98364 Burkina Faso,BFA,2008,524.59015 Burkina Faso,BFA,2009,542.8571 Burkina Faso,BFA,2010,517.2414 -Burkina Faso,BFA,2011,555.5556 +Burkina Faso,BFA,2011,555.55554 Burkina Faso,BFA,2012,546.875 Burkina Faso,BFA,2013,546.6667 -Burkina Faso,BFA,2014,579.5454 +Burkina Faso,BFA,2014,579.5455 Burkina Faso,BFA,2015,594.05945 Burkina Faso,BFA,2016,555.55554 Burkina Faso,BFA,2017,562.5 -Burkina Faso,BFA,2018,518.1818 -Burkina Faso,BFA,2019,487.49997 -Burkina Faso,BFA,2020,472.97296 -Burkina Faso,BFA,2021,473.68423 -Burkina Faso,BFA,2022,467.5325 +Burkina Faso,BFA,2018,566.8449 +Burkina Faso,BFA,2019,556.25 +Burkina Faso,BFA,2020,547.1698 +Burkina Faso,BFA,2021,549.7076 +Burkina Faso,BFA,2022,568.0473 +Burkina Faso,BFA,2023,554.91327 Burundi,BDI,2000,0 Burundi,BDI,2001,0 Burundi,BDI,2002,0 @@ -860,17 +866,18 @@ Burundi,BDI,2010,62.5 Burundi,BDI,2011,66.666664 Burundi,BDI,2012,0 Burundi,BDI,2013,58.82353 -Burundi,BDI,2014,111.111115 -Burundi,BDI,2015,111.111115 -Burundi,BDI,2016,55.555557 +Burundi,BDI,2014,111.11111 +Burundi,BDI,2015,111.11111 +Burundi,BDI,2016,55.555553 Burundi,BDI,2017,178.57143 -Burundi,BDI,2018,259.25928 -Burundi,BDI,2019,250.00002 -Burundi,BDI,2020,212.12122 -Burundi,BDI,2021,250.00002 -Burundi,BDI,2022,250.00002 -Cambodia,KHM,2000,545.4546 -Cambodia,KHM,2001,571.4286 +Burundi,BDI,2018,228.57143 +Burundi,BDI,2019,230.76924 +Burundi,BDI,2020,189.1892 +Burundi,BDI,2021,230.76924 +Burundi,BDI,2022,230.76924 +Burundi,BDI,2023,230.76924 +Cambodia,KHM,2000,545.4545 +Cambodia,KHM,2001,571.4285 Cambodia,KHM,2002,631.579 Cambodia,KHM,2003,609.375 Cambodia,KHM,2004,618.4211 @@ -878,8 +885,8 @@ Cambodia,KHM,2005,621.0526 Cambodia,KHM,2006,623.93164 Cambodia,KHM,2007,624.1611 Cambodia,KHM,2008,628.37836 -Cambodia,KHM,2009,634.9206 -Cambodia,KHM,2010,633.6634 +Cambodia,KHM,2009,634.92065 +Cambodia,KHM,2010,633.6633 Cambodia,KHM,2011,622.64154 Cambodia,KHM,2012,424.65753 Cambodia,KHM,2013,314.91714 @@ -887,60 +894,64 @@ Cambodia,KHM,2014,337.62057 Cambodia,KHM,2015,480.34937 Cambodia,KHM,2016,460.733 Cambodia,KHM,2017,512.40875 -Cambodia,KHM,2018,371.98068 -Cambodia,KHM,2019,447.3386 -Cambodia,KHM,2020,437.6392 -Cambodia,KHM,2021,415.11624 -Cambodia,KHM,2022,417.70712 -Cameroon,CMR,2000,31.884056 +Cambodia,KHM,2018,360.9813 +Cambodia,KHM,2019,436.9469 +Cambodia,KHM,2020,438.6161 +Cambodia,KHM,2021,356.1508 +Cambodia,KHM,2022,305.16434 +Cambodia,KHM,2023,470.58823 +Cambodia,KHM,2024,497.45547 +Cameroon,CMR,2000,31.884058 Cameroon,CMR,2001,37.037037 Cameroon,CMR,2002,45.871563 -Cameroon,CMR,2003,52.05479 +Cameroon,CMR,2003,52.054794 Cameroon,CMR,2004,54.05405 Cameroon,CMR,2005,60.60606 -Cameroon,CMR,2006,120.553375 +Cameroon,CMR,2006,120.55337 Cameroon,CMR,2007,199.61612 Cameroon,CMR,2008,193.26242 Cameroon,CMR,2009,222.99652 Cameroon,CMR,2010,206.48465 -Cameroon,CMR,2011,192.17688 +Cameroon,CMR,2011,192.17686 Cameroon,CMR,2012,167.5875 -Cameroon,CMR,2013,239.7476 +Cameroon,CMR,2013,239.74762 Cameroon,CMR,2014,281.5675 -Cameroon,CMR,2015,336.68344 +Cameroon,CMR,2015,336.6834 Cameroon,CMR,2016,299.57806 Cameroon,CMR,2017,288.19873 Cameroon,CMR,2018,324.35596 -Cameroon,CMR,2019,295.69266 -Cameroon,CMR,2020,314.44446 -Cameroon,CMR,2021,298.75003 -Cameroon,CMR,2022,305.4187 +Cameroon,CMR,2019,305.55557 +Cameroon,CMR,2020,320.45453 +Cameroon,CMR,2021,298.45422 +Cameroon,CMR,2022,280.9816 +Cameroon,CMR,2023,285.7143 Canada,CAN,2000,245.6858 -Canada,CAN,2001,255.64856 -Canada,CAN,2002,245.11913 +Canada,CAN,2001,255.64854 +Canada,CAN,2002,245.11914 Canada,CAN,2003,251.63614 -Canada,CAN,2004,235.33838 -Canada,CAN,2005,225.01207 +Canada,CAN,2004,235.33836 +Canada,CAN,2005,225.01208 Canada,CAN,2006,216.50806 Canada,CAN,2007,227.7551 -Canada,CAN,2008,213.47485 +Canada,CAN,2008,213.47484 Canada,CAN,2009,204.32664 -Canada,CAN,2010,214.78346 -Canada,CAN,2011,206.18233 +Canada,CAN,2010,214.78345 +Canada,CAN,2011,206.18234 Canada,CAN,2012,193.38927 Canada,CAN,2013,186.96335 Canada,CAN,2014,188.86014 -Canada,CAN,2015,184.36496 +Canada,CAN,2015,184.36497 Canada,CAN,2016,180.678 Canada,CAN,2017,172.01057 -Canada,CAN,2018,170.79106 +Canada,CAN,2018,170.79108 Canada,CAN,2019,169.916 -Canada,CAN,2020,160.64651 -Canada,CAN,2021,164.04594 -Canada,CAN,2022,161.43408 -Canada,CAN,2023,170.04251 -Cape Verde,CPV,2000,600.00006 -Cape Verde,CPV,2001,647.0588 +Canada,CAN,2020,160.49854 +Canada,CAN,2021,163.00218 +Canada,CAN,2022,156.79489 +Canada,CAN,2023,165.1479 +Canada,CAN,2024,174.81425 +Cape Verde,CPV,2000,600 +Cape Verde,CPV,2001,647.05884 Cape Verde,CPV,2002,631.5789 Cape Verde,CPV,2003,684.2105 Cape Verde,CPV,2004,636.36365 @@ -948,11 +959,11 @@ Cape Verde,CPV,2005,625.00006 Cape Verde,CPV,2006,640 Cape Verde,CPV,2007,629.62964 Cape Verde,CPV,2008,620.6897 -Cape Verde,CPV,2009,645.16125 -Cape Verde,CPV,2010,647.0588 +Cape Verde,CPV,2009,645.1613 +Cape Verde,CPV,2010,647.05884 Cape Verde,CPV,2011,594.5946 Cape Verde,CPV,2012,526.3158 -Cape Verde,CPV,2013,512.19507 +Cape Verde,CPV,2013,512.1951 Cape Verde,CPV,2014,511.6279 Cape Verde,CPV,2015,511.6279 Cape Verde,CPV,2016,533.3333 @@ -960,8 +971,8 @@ Cape Verde,CPV,2017,534.8837 Cape Verde,CPV,2018,511.6279 Cape Verde,CPV,2019,545.4545 Cape Verde,CPV,2020,547.6191 -Cape Verde,CPV,2021,547.6191 -Cape Verde,CPV,2022,558.1395 +Cape Verde,CPV,2021,534.8837 +Cape Verde,CPV,2022,480 Cayman Islands,CYM,2000,659.0909 Cayman Islands,CYM,2001,659.57446 Cayman Islands,CYM,2002,666.6667 @@ -969,22 +980,23 @@ Cayman Islands,CYM,2003,666.6667 Cayman Islands,CYM,2004,666.6667 Cayman Islands,CYM,2005,666.6667 Cayman Islands,CYM,2006,660.7143 -Cayman Islands,CYM,2007,650 -Cayman Islands,CYM,2008,661.29034 +Cayman Islands,CYM,2007,649.99994 +Cayman Islands,CYM,2008,661.2903 Cayman Islands,CYM,2009,650.79364 -Cayman Islands,CYM,2010,661.29034 +Cayman Islands,CYM,2010,661.2903 Cayman Islands,CYM,2011,650.79364 Cayman Islands,CYM,2012,655.7377 -Cayman Islands,CYM,2013,661.29034 -Cayman Islands,CYM,2014,661.29034 -Cayman Islands,CYM,2015,656.24994 +Cayman Islands,CYM,2013,661.2903 +Cayman Islands,CYM,2014,661.2903 +Cayman Islands,CYM,2015,656.25 Cayman Islands,CYM,2016,656.7164 -Cayman Islands,CYM,2017,647.0588 -Cayman Islands,CYM,2018,641.7911 -Cayman Islands,CYM,2019,638.8889 +Cayman Islands,CYM,2017,647.05884 +Cayman Islands,CYM,2018,641.791 +Cayman Islands,CYM,2019,638.88885 Cayman Islands,CYM,2020,637.68115 Cayman Islands,CYM,2021,642.8571 Cayman Islands,CYM,2022,642.8571 +Cayman Islands,CYM,2023,642.8571 Central African Republic,CAF,2000,100 Central African Republic,CAF,2001,181.81818 Central African Republic,CAF,2002,181.81818 @@ -996,8 +1008,8 @@ Central African Republic,CAF,2007,62.5 Central African Republic,CAF,2008,62.5 Central African Republic,CAF,2009,62.5 Central African Republic,CAF,2010,125 -Central African Republic,CAF,2011,111.111115 -Central African Republic,CAF,2012,111.111115 +Central African Republic,CAF,2011,111.11111 +Central African Republic,CAF,2012,111.11111 Central African Republic,CAF,2013,0 Central African Republic,CAF,2014,0 Central African Republic,CAF,2015,0 @@ -1017,92 +1029,95 @@ Chad,TCD,2005,666.6667 Chad,TCD,2006,666.6667 Chad,TCD,2007,666.6667 Chad,TCD,2008,642.8572 -Chad,TCD,2009,647.0588 +Chad,TCD,2009,647.05884 Chad,TCD,2010,684.2105 -Chad,TCD,2011,650 +Chad,TCD,2011,649.99994 Chad,TCD,2012,625.00006 Chad,TCD,2013,642.8572 Chad,TCD,2014,633.3333 Chad,TCD,2015,633.3333 Chad,TCD,2016,612.9032 Chad,TCD,2017,625 -Chad,TCD,2018,625 -Chad,TCD,2019,606.0606 +Chad,TCD,2018,617.64703 +Chad,TCD,2019,628.5714 Chad,TCD,2020,628.5714 -Chad,TCD,2021,628.5714 -Chad,TCD,2022,628.5714 -Chile,CHL,2000,380.4627 -Chile,CHL,2001,327.6711 -Chile,CHL,2002,323.34198 -Chile,CHL,2003,343.37616 -Chile,CHL,2004,388.6149 -Chile,CHL,2005,349.46664 -Chile,CHL,2006,349.82266 -Chile,CHL,2007,450.51804 -Chile,CHL,2008,442.61133 -Chile,CHL,2009,433.05698 -Chile,CHL,2010,468.9324 -Chile,CHL,2011,498.2269 -Chile,CHL,2012,525.0912 -Chile,CHL,2013,549.9487 -Chile,CHL,2014,498.78485 -Chile,CHL,2015,483.62372 -Chile,CHL,2016,524.26385 -Chile,CHL,2017,474.93936 -Chile,CHL,2018,452.4368 -Chile,CHL,2019,453.6762 -Chile,CHL,2020,436.3987 -Chile,CHL,2021,442.39972 -Chile,CHL,2022,353.52075 -Chile,CHL,2023,291.1135 +Chad,TCD,2021,627.907 +Chad,TCD,2022,615.38464 +Chile,CHL,2000,388.1323 +Chile,CHL,2001,336.48392 +Chile,CHL,2002,330.1105 +Chile,CHL,2003,355.35062 +Chile,CHL,2004,391.7647 +Chile,CHL,2005,347.56796 +Chile,CHL,2006,355.3779 +Chile,CHL,2007,456.93204 +Chile,CHL,2008,454.42316 +Chile,CHL,2009,435.97815 +Chile,CHL,2010,481.65976 +Chile,CHL,2011,493.06927 +Chile,CHL,2012,531.4072 +Chile,CHL,2013,551.75726 +Chile,CHL,2014,493.65826 +Chile,CHL,2015,491.38965 +Chile,CHL,2016,496.3846 +Chile,CHL,2017,487.26718 +Chile,CHL,2018,470.27887 +Chile,CHL,2019,456.53726 +Chile,CHL,2020,436.82697 +Chile,CHL,2021,425.45624 +Chile,CHL,2022,372.80313 +Chile,CHL,2023,301.92767 +Chile,CHL,2024,265.51648 China,CHN,2000,783.31366 -China,CHN,2001,762.73474 -China,CHN,2002,773.1633 +China,CHN,2001,762.7347 +China,CHN,2002,773.1634 China,CHN,2003,790.7861 China,CHN,2004,778.35614 -China,CHN,2005,781.9307 +China,CHN,2005,781.93066 China,CHN,2006,786.51086 China,CHN,2007,785.12524 China,CHN,2008,746.9477 China,CHN,2009,753.02856 China,CHN,2010,740.6397 China,CHN,2011,752.2311 -China,CHN,2012,720.1506 +China,CHN,2012,720.15063 China,CHN,2013,717.38873 -China,CHN,2014,693.72754 +China,CHN,2014,693.7276 China,CHN,2015,667.7146 China,CHN,2016,651.6657 China,CHN,2017,644.1666 China,CHN,2018,636.8756 China,CHN,2019,619.61664 China,CHN,2020,606.25757 -China,CHN,2021,598.85565 -China,CHN,2022,585.82074 -China,CHN,2023,582.31696 -Colombia,COL,2000,189.20815 -Colombia,COL,2001,197.72833 +China,CHN,2021,599.0045 +China,CHN,2022,586.7002 +China,CHN,2023,583.608 +China,CHN,2024,559.5517 +Colombia,COL,2000,189.20813 +Colombia,COL,2001,197.72832 Colombia,COL,2002,183.40416 Colombia,COL,2003,172.07791 -Colombia,COL,2004,148.38841 +Colombia,COL,2004,148.3884 Colombia,COL,2005,159.99199 Colombia,COL,2006,158.98012 -Colombia,COL,2007,155.13779 +Colombia,COL,2007,155.1378 Colombia,COL,2008,137.52704 -Colombia,COL,2009,211.62997 +Colombia,COL,2009,211.62996 Colombia,COL,2010,229.45375 -Colombia,COL,2011,151.37762 -Colombia,COL,2012,179.03584 +Colombia,COL,2011,151.37764 +Colombia,COL,2012,179.03583 Colombia,COL,2013,236.02231 Colombia,COL,2014,236.73007 Colombia,COL,2015,275.3623 Colombia,COL,2016,259.55103 -Colombia,COL,2017,166.17815 +Colombia,COL,2017,166.17816 Colombia,COL,2018,187.44406 Colombia,COL,2019,240.92409 -Colombia,COL,2020,273.20514 -Colombia,COL,2021,196.28049 -Colombia,COL,2022,214.88087 -Colombia,COL,2023,259.51117 +Colombia,COL,2020,271.25403 +Colombia,COL,2021,195.82741 +Colombia,COL,2022,202.1482 +Colombia,COL,2023,268.96704 +Colombia,COL,2024,285.7959 Comoros,COM,2000,666.6667 Comoros,COM,2001,666.6667 Comoros,COM,2002,666.6667 @@ -1123,20 +1138,21 @@ Comoros,COM,2016,666.6667 Comoros,COM,2017,666.6666 Comoros,COM,2018,700 Comoros,COM,2019,636.36365 -Comoros,COM,2020,692.3078 -Comoros,COM,2021,692.3078 +Comoros,COM,2020,692.30774 +Comoros,COM,2021,692.30774 Comoros,COM,2022,642.8572 +Comoros,COM,2023,642.8572 Congo,COG,2000,34.482758 -Congo,COG,2001,30.30303 +Congo,COG,2001,30.303028 Congo,COG,2002,25.641026 Congo,COG,2003,150 Congo,COG,2004,175 Congo,COG,2005,186.04651 Congo,COG,2006,177.77777 Congo,COG,2007,175 -Congo,COG,2008,195.65219 +Congo,COG,2008,195.65218 Congo,COG,2009,351.85184 -Congo,COG,2010,423.07693 +Congo,COG,2010,423.07697 Congo,COG,2011,375 Congo,COG,2012,396.4497 Congo,COG,2013,411.76468 @@ -1144,11 +1160,12 @@ Congo,COG,2014,427.74567 Congo,COG,2015,543.4783 Congo,COG,2016,561.2245 Congo,COG,2017,545.7627 -Congo,COG,2018,592.23303 -Congo,COG,2019,693.9891 -Congo,COG,2020,691.90607 -Congo,COG,2021,697.20105 -Congo,COG,2022,700 +Congo,COG,2018,596.7213 +Congo,COG,2019,658.03107 +Congo,COG,2020,665.82916 +Congo,COG,2021,671.56866 +Congo,COG,2022,709.3496 +Congo,COG,2023,713.7331 Cook Islands,COK,2000,666.6667 Cook Islands,COK,2001,666.6667 Cook Islands,COK,2002,666.6667 @@ -1172,120 +1189,104 @@ Cook Islands,COK,2019,500 Cook Islands,COK,2020,500 Cook Islands,COK,2021,250 Cook Islands,COK,2022,250 -Costa Rica,CRI,2000,30.612242 +Costa Rica,CRI,2000,30.612244 Costa Rica,CRI,2001,34.782608 Costa Rica,CRI,2002,36.38814 Costa Rica,CRI,2003,39.726025 -Costa Rica,CRI,2004,31.32832 +Costa Rica,CRI,2004,31.328321 Costa Rica,CRI,2005,47.61905 -Costa Rica,CRI,2006,67.28538 +Costa Rica,CRI,2006,67.285385 Costa Rica,CRI,2007,77.951004 Costa Rica,CRI,2008,72.34043 -Costa Rica,CRI,2009,58.441566 -Costa Rica,CRI,2010,69.40063 -Costa Rica,CRI,2011,84.10257 -Costa Rica,CRI,2012,80.11869 -Costa Rica,CRI,2013,103.042206 -Costa Rica,CRI,2014,93.41199 +Costa Rica,CRI,2009,58.441563 +Costa Rica,CRI,2010,69.400635 +Costa Rica,CRI,2011,84.10256 +Costa Rica,CRI,2012,80.1187 +Costa Rica,CRI,2013,103.0422 +Costa Rica,CRI,2014,93.411995 Costa Rica,CRI,2015,33.426186 Costa Rica,CRI,2016,38.674034 Costa Rica,CRI,2017,28.368793 -Costa Rica,CRI,2018,34.965034 -Costa Rica,CRI,2019,31.496063 -Costa Rica,CRI,2020,27.280478 -Costa Rica,CRI,2021,26.128267 -Costa Rica,CRI,2022,26.463512 -Costa Rica,CRI,2023,53.377815 +Costa Rica,CRI,2018,34.96504 +Costa Rica,CRI,2019,31.523643 +Costa Rica,CRI,2020,27.303753 +Costa Rica,CRI,2021,25.92302 +Costa Rica,CRI,2022,24.115757 +Costa Rica,CRI,2023,24.765158 +Costa Rica,CRI,2024,63.004845 Cote d'Ivoire,CIV,2000,364.5833 Cote d'Ivoire,CIV,2001,362.7049 Cote d'Ivoire,CIV,2002,385.63327 Cote d'Ivoire,CIV,2003,368.11023 -Cote d'Ivoire,CIV,2004,383.63632 -Cote d'Ivoire,CIV,2005,416.2257 -Cote d'Ivoire,CIV,2006,409.57443 +Cote d'Ivoire,CIV,2004,383.63635 +Cote d'Ivoire,CIV,2005,416.22574 +Cote d'Ivoire,CIV,2006,409.57446 Cote d'Ivoire,CIV,2007,380.78296 Cote d'Ivoire,CIV,2008,378.23834 -Cote d'Ivoire,CIV,2009,362.39313 +Cote d'Ivoire,CIV,2009,362.39316 Cote d'Ivoire,CIV,2010,413.4454 Cote d'Ivoire,CIV,2011,399.67108 Cote d'Ivoire,CIV,2012,425.7143 Cote d'Ivoire,CIV,2013,446.92004 Cote d'Ivoire,CIV,2014,440.14508 -Cote d'Ivoire,CIV,2015,475.86203 -Cote d'Ivoire,CIV,2016,475.07336 -Cote d'Ivoire,CIV,2017,446.64035 -Cote d'Ivoire,CIV,2018,399.40244 +Cote d'Ivoire,CIV,2015,475.86206 +Cote d'Ivoire,CIV,2016,475.07333 +Cote d'Ivoire,CIV,2017,446.64032 +Cote d'Ivoire,CIV,2018,399.4024 Cote d'Ivoire,CIV,2019,382.7392 Cote d'Ivoire,CIV,2020,396.09235 -Cote d'Ivoire,CIV,2021,431.71802 -Cote d'Ivoire,CIV,2022,393.8849 -Croatia,HRV,1990,360.95346 -Croatia,HRV,1991,242.42424 -Croatia,HRV,1992,332.5843 -Croatia,HRV,1993,323.89935 -Croatia,HRV,1994,217.3913 -Croatia,HRV,1995,252.1645 -Croatia,HRV,1996,186.71452 -Croatia,HRV,1997,283.05582 -Croatia,HRV,1998,310.31467 -Croatia,HRV,1999,288.58026 +Cote d'Ivoire,CIV,2021,431.71805 +Cote d'Ivoire,CIV,2022,413.16418 +Cote d'Ivoire,CIV,2023,393.531 Croatia,HRV,2000,302.40857 Croatia,HRV,2001,306.6667 Croatia,HRV,2002,370.37036 -Croatia,HRV,2003,414.13376 +Croatia,HRV,2003,414.13373 Croatia,HRV,2004,315.06848 Croatia,HRV,2005,336.39844 Croatia,HRV,2006,347.01782 -Croatia,HRV,2007,420.25516 -Croatia,HRV,2008,382.62915 -Croatia,HRV,2009,301.87265 +Croatia,HRV,2007,420.2552 +Croatia,HRV,2008,382.62912 +Croatia,HRV,2009,301.87268 Croatia,HRV,2010,268.42462 -Croatia,HRV,2011,374.88873 -Croatia,HRV,2012,351.93948 +Croatia,HRV,2011,374.8887 +Croatia,HRV,2012,351.93945 Croatia,HRV,2013,256.09756 -Croatia,HRV,2014,226.35887 +Croatia,HRV,2014,226.35889 Croatia,HRV,2015,274.91104 Croatia,HRV,2016,275.39682 Croatia,HRV,2017,255.73491 -Croatia,HRV,2018,201.18344 +Croatia,HRV,2018,201.18343 Croatia,HRV,2019,242.68774 -Croatia,HRV,2020,226.75739 +Croatia,HRV,2020,226.75737 Croatia,HRV,2021,210.45667 -Croatia,HRV,2022,242.27528 -Croatia,HRV,2023,204.96161 +Croatia,HRV,2022,244.68085 +Croatia,HRV,2023,197.33025 +Croatia,HRV,2024,174.47917 Cuba,CUB,2000,630.7385 -Cuba,CUB,2001,632.0262 -Cuba,CUB,2002,630.9751 +Cuba,CUB,2001,632.0261 +Cuba,CUB,2002,630.97516 Cuba,CUB,2003,637.5711 Cuba,CUB,2004,637.468 Cuba,CUB,2005,648.8599 Cuba,CUB,2006,650.27325 Cuba,CUB,2007,650.1987 Cuba,CUB,2008,645.9276 -Cuba,CUB,2009,645.79816 -Cuba,CUB,2010,648.8505 +Cuba,CUB,2009,645.7981 +Cuba,CUB,2010,648.8506 Cuba,CUB,2011,648.45074 Cuba,CUB,2012,645.68634 Cuba,CUB,2013,639.4805 Cuba,CUB,2014,643.335 -Cuba,CUB,2015,643.9432 -Cuba,CUB,2016,643.3091 +Cuba,CUB,2015,643.9431 +Cuba,CUB,2016,643.309 Cuba,CUB,2017,645.6004 -Cuba,CUB,2018,645.8733 -Cuba,CUB,2019,640.4441 -Cuba,CUB,2020,638.32025 -Cuba,CUB,2021,637.3811 -Cuba,CUB,2022,637.6096 -Cyprus,CYP,1990,659.89844 -Cyprus,CYP,1991,658.65393 -Cyprus,CYP,1992,658.43616 -Cyprus,CYP,1993,656.3707 -Cyprus,CYP,1994,656.7164 -Cyprus,CYP,1995,660 -Cyprus,CYP,1996,659.0039 -Cyprus,CYP,1997,659.34064 -Cyprus,CYP,1998,658.86285 -Cyprus,CYP,1999,659.23566 +Cuba,CUB,2018,647.1154 +Cuba,CUB,2019,641.0628 +Cuba,CUB,2020,638.991 +Cuba,CUB,2021,638.81104 +Cuba,CUB,2022,640.2556 +Cuba,CUB,2023,638.97974 Cyprus,CYP,2000,658.7537 Cyprus,CYP,2001,659.1549 Cyprus,CYP,2002,656.99207 @@ -1293,91 +1294,74 @@ Cyprus,CYP,2003,656.7901 Cyprus,CYP,2004,657.1429 Cyprus,CYP,2005,657.53424 Cyprus,CYP,2006,658.0645 -Cyprus,CYP,2007,657.08417 -Cyprus,CYP,2008,656.80475 -Cyprus,CYP,2009,656.43 +Cyprus,CYP,2007,657.0842 +Cyprus,CYP,2008,656.8047 +Cyprus,CYP,2009,656.42993 Cyprus,CYP,2010,649.15576 -Cyprus,CYP,2011,638.2114 -Cyprus,CYP,2012,622.88135 +Cyprus,CYP,2011,638.21136 +Cyprus,CYP,2012,622.8814 Cyprus,CYP,2013,610.7226 Cyprus,CYP,2014,612.9032 -Cyprus,CYP,2015,603.5243 -Cyprus,CYP,2016,603.272 +Cyprus,CYP,2015,603.52423 +Cyprus,CYP,2016,603.27203 Cyprus,CYP,2017,606 -Cyprus,CYP,2018,598.8143 +Cyprus,CYP,2018,598.8142 Cyprus,CYP,2019,596.11646 Cyprus,CYP,2020,581.4433 -Cyprus,CYP,2021,564.4531 +Cyprus,CYP,2021,564.4532 Cyprus,CYP,2022,555.13306 -Cyprus,CYP,2023,534.3229 -Czechia,CZE,1990,777.74207 -Czechia,CZE,1991,779.1045 -Czechia,CZE,1992,766.9772 -Czechia,CZE,1993,758.4797 -Czechia,CZE,1994,746.9177 -Czechia,CZE,1995,756.025 -Czechia,CZE,1996,749.33417 -Czechia,CZE,1997,757.67017 -Czechia,CZE,1998,749.3035 -Czechia,CZE,1999,736.9488 +Cyprus,CYP,2023,534.8399 +Cyprus,CYP,2024,512.2378 Czechia,CZE,2000,767.0781 -Czechia,CZE,2001,751.81885 +Czechia,CZE,2001,751.8189 Czechia,CZE,2002,699.3421 -Czechia,CZE,2003,651.8904 +Czechia,CZE,2003,651.8903 Czechia,CZE,2004,640.1718 -Czechia,CZE,2005,646.64954 -Czechia,CZE,2006,634.1142 +Czechia,CZE,2005,646.6496 +Czechia,CZE,2006,634.11426 Czechia,CZE,2007,652.273 Czechia,CZE,2008,627.5216 Czechia,CZE,2009,603.13574 -Czechia,CZE,2010,593.64 +Czechia,CZE,2010,593.63995 Czechia,CZE,2011,583.5642 -Czechia,CZE,2012,552.7835 +Czechia,CZE,2012,552.78357 Czechia,CZE,2013,527.5288 Czechia,CZE,2014,530.83075 Czechia,CZE,2015,553.15796 Czechia,CZE,2016,573.90247 Czechia,CZE,2017,542.3274 Czechia,CZE,2018,532.5887 -Czechia,CZE,2019,503.4998 +Czechia,CZE,2019,503.49976 Czechia,CZE,2020,460.1748 Czechia,CZE,2021,471.7906 -Czechia,CZE,2022,488.29712 -Czechia,CZE,2023,449.72433 +Czechia,CZE,2022,488.3554 +Czechia,CZE,2023,442.55377 +Czechia,CZE,2024,413.85922 Democratic Republic of Congo,COD,2000,23.569023 Democratic Republic of Congo,COD,2001,25.33784 -Democratic Republic of Congo,COD,2002,24.752478 +Democratic Republic of Congo,COD,2002,24.752476 Democratic Republic of Congo,COD,2003,24.54992 -Democratic Republic of Congo,COD,2004,25.64103 -Democratic Republic of Congo,COD,2005,24.55662 -Democratic Republic of Congo,COD,2006,27.999998 -Democratic Republic of Congo,COD,2007,30.651339 +Democratic Republic of Congo,COD,2004,25.641027 +Democratic Republic of Congo,COD,2005,24.556618 +Democratic Republic of Congo,COD,2006,28 +Democratic Republic of Congo,COD,2007,30.65134 Democratic Republic of Congo,COD,2008,31.957388 Democratic Republic of Congo,COD,2009,30.729834 -Democratic Republic of Congo,COD,2010,30.651339 -Democratic Republic of Congo,COD,2011,30.690535 +Democratic Republic of Congo,COD,2010,30.65134 +Democratic Republic of Congo,COD,2011,30.690536 Democratic Republic of Congo,COD,2012,28.68318 Democratic Republic of Congo,COD,2013,27.644232 -Democratic Republic of Congo,COD,2014,28.089888 +Democratic Republic of Congo,COD,2014,28.08989 Democratic Republic of Congo,COD,2015,28.856825 Democratic Republic of Congo,COD,2016,28.291622 -Democratic Republic of Congo,COD,2017,28.183716 +Democratic Republic of Congo,COD,2017,28.183718 Democratic Republic of Congo,COD,2018,29.878618 -Democratic Republic of Congo,COD,2019,28.293545 -Democratic Republic of Congo,COD,2020,25.083612 -Democratic Republic of Congo,COD,2021,24.456524 -Democratic Republic of Congo,COD,2022,24.456524 -Denmark,DNK,1990,889.74554 -Denmark,DNK,1991,894.9245 -Denmark,DNK,1992,872.2222 -Denmark,DNK,1993,863.82477 -Denmark,DNK,1994,850.91626 -Denmark,DNK,1995,813.0125 -Denmark,DNK,1996,817.1599 -Denmark,DNK,1997,761.2143 -Denmark,DNK,1998,709.18866 -Denmark,DNK,1999,673.2955 -Denmark,DNK,2000,627.894 +Democratic Republic of Congo,COD,2019,28.262678 +Democratic Republic of Congo,COD,2020,25.579535 +Democratic Republic of Congo,COD,2021,26.038002 +Democratic Republic of Congo,COD,2022,27.397259 +Democratic Republic of Congo,COD,2023,27.044025 +Denmark,DNK,2000,627.89404 Denmark,DNK,2001,630.7938 Denmark,DNK,2002,618.6701 Denmark,DNK,2003,651.51184 @@ -1388,32 +1372,33 @@ Denmark,DNK,2007,597.0874 Denmark,DNK,2008,577.3026 Denmark,DNK,2009,582.89655 Denmark,DNK,2010,540.3871 -Denmark,DNK,2011,486.4711 +Denmark,DNK,2011,486.47107 Denmark,DNK,2012,431.14166 Denmark,DNK,2013,472.40683 -Denmark,DNK,2014,400.74902 +Denmark,DNK,2014,400.74905 Denmark,DNK,2015,313.75955 -Denmark,DNK,2016,360.85526 +Denmark,DNK,2016,360.8553 Denmark,DNK,2017,282.8479 Denmark,DNK,2018,296.92157 Denmark,DNK,2019,204.29008 -Denmark,DNK,2020,188.94333 -Denmark,DNK,2021,222.96477 -Denmark,DNK,2022,201.77702 -Denmark,DNK,2023,151.6503 +Denmark,DNK,2020,188.94331 +Denmark,DNK,2021,222.96478 +Denmark,DNK,2022,201.20033 +Denmark,DNK,2023,149.80719 +Denmark,DNK,2024,143.30042 Djibouti,DJI,2000,666.6666 Djibouti,DJI,2001,684.2105 -Djibouti,DJI,2002,650 -Djibouti,DJI,2003,650 +Djibouti,DJI,2002,649.99994 +Djibouti,DJI,2003,649.99994 Djibouti,DJI,2004,636.36365 Djibouti,DJI,2005,666.6666 -Djibouti,DJI,2006,645.16125 -Djibouti,DJI,2007,656.24994 -Djibouti,DJI,2008,647.0588 +Djibouti,DJI,2006,645.1613 +Djibouti,DJI,2007,656.25 +Djibouti,DJI,2008,647.05884 Djibouti,DJI,2009,657.1429 -Djibouti,DJI,2010,657.8947 -Djibouti,DJI,2011,666.6666 -Djibouti,DJI,2012,666.6666 +Djibouti,DJI,2010,657.8948 +Djibouti,DJI,2011,666.6667 +Djibouti,DJI,2012,666.6667 Djibouti,DJI,2013,651.1628 Djibouti,DJI,2014,700 Djibouti,DJI,2015,666.6666 @@ -1421,66 +1406,95 @@ Djibouti,DJI,2016,666.6666 Djibouti,DJI,2017,666.6667 Djibouti,DJI,2018,600 Djibouti,DJI,2019,666.6666 -Djibouti,DJI,2020,692.3078 -Djibouti,DJI,2021,692.3078 -Djibouti,DJI,2022,692.3078 +Djibouti,DJI,2020,692.30774 +Djibouti,DJI,2021,450 +Djibouti,DJI,2022,450 +Djibouti,DJI,2023,450 Dominica,DMA,2000,375 Dominica,DMA,2001,375 Dominica,DMA,2002,428.5714 Dominica,DMA,2003,375 -Dominica,DMA,2004,454.54544 -Dominica,DMA,2005,499.99997 -Dominica,DMA,2006,499.99997 -Dominica,DMA,2007,583.3333 -Dominica,DMA,2008,538.4616 -Dominica,DMA,2009,538.4616 -Dominica,DMA,2010,571.4286 -Dominica,DMA,2011,466.6667 +Dominica,DMA,2004,454.54547 +Dominica,DMA,2005,500 +Dominica,DMA,2006,500 +Dominica,DMA,2007,583.3334 +Dominica,DMA,2008,538.46155 +Dominica,DMA,2009,538.46155 +Dominica,DMA,2010,571.4285 +Dominica,DMA,2011,466.66666 Dominica,DMA,2012,533.3333 -Dominica,DMA,2013,466.6667 +Dominica,DMA,2013,466.66666 Dominica,DMA,2014,533.3333 -Dominica,DMA,2015,600.00006 +Dominica,DMA,2015,600 Dominica,DMA,2016,500 -Dominica,DMA,2017,538.4616 +Dominica,DMA,2017,538.46155 Dominica,DMA,2018,545.4545 -Dominica,DMA,2019,571.4286 -Dominica,DMA,2020,500 -Dominica,DMA,2021,500 -Dominica,DMA,2022,529.4118 -Dominican Republic,DOM,2000,615.6803 +Dominica,DMA,2019,571.4285 +Dominica,DMA,2020,571.4285 +Dominica,DMA,2021,571.4285 +Dominica,DMA,2022,600 +Dominica,DMA,2023,600 +Dominican Republic,DOM,2000,615.68024 Dominican Republic,DOM,2001,619.9377 -Dominican Republic,DOM,2002,621.5828 +Dominican Republic,DOM,2002,621.58276 Dominican Republic,DOM,2003,612.12976 -Dominican Republic,DOM,2004,584.02655 +Dominican Republic,DOM,2004,584.0266 Dominican Republic,DOM,2005,574.9625 -Dominican Republic,DOM,2006,588.2353 -Dominican Republic,DOM,2007,588.19495 +Dominican Republic,DOM,2006,588.23535 +Dominican Republic,DOM,2007,588.1949 Dominican Republic,DOM,2008,601.6427 Dominican Republic,DOM,2009,593.4545 -Dominican Republic,DOM,2010,587.9567 +Dominican Republic,DOM,2010,587.95667 Dominican Republic,DOM,2011,590.1015 Dominican Republic,DOM,2012,582.24854 Dominican Republic,DOM,2013,571.6705 -Dominican Republic,DOM,2014,593.75006 -Dominican Republic,DOM,2015,606.11115 +Dominican Republic,DOM,2014,593.75 +Dominican Republic,DOM,2015,606.1111 Dominican Republic,DOM,2016,589.9893 Dominican Republic,DOM,2017,550.5208 -Dominican Republic,DOM,2018,561.9959 -Dominican Republic,DOM,2019,578.5124 -Dominican Republic,DOM,2020,536.3096 -Dominican Republic,DOM,2021,578.8452 -Dominican Republic,DOM,2022,580.7799 +Dominican Republic,DOM,2018,566.32654 +Dominican Republic,DOM,2019,582.7067 +Dominican Republic,DOM,2020,581.3237 +Dominican Republic,DOM,2021,558.56274 +Dominican Republic,DOM,2022,580.0631 +Dominican Republic,DOM,2023,579.3974 +Dominican Republic,DOM,2024,566.6918 +EU (Ember),,2000,419.0956 +EU (Ember),,2001,409.1615 +EU (Ember),,2002,419.65817 +EU (Ember),,2003,424.2407 +EU (Ember),,2004,412.04453 +EU (Ember),,2005,409.94046 +EU (Ember),,2006,408.7229 +EU (Ember),,2007,413.10452 +EU (Ember),,2008,392.42636 +EU (Ember),,2009,382.45203 +EU (Ember),,2010,369.40326 +EU (Ember),,2011,376.72073 +EU (Ember),,2012,371.948 +EU (Ember),,2013,357.1669 +EU (Ember),,2014,342.94556 +EU (Ember),,2015,348.39474 +EU (Ember),,2016,341.45538 +EU (Ember),,2017,340.3681 +EU (Ember),,2018,321.30505 +EU (Ember),,2019,287.34488 +EU (Ember),,2020,261.23105 +EU (Ember),,2021,272.61124 +EU (Ember),,2022,291.70627 +EU (Ember),,2023,237.48735 +EU (Ember),,2024,213.3101 East Timor,TLS,2003,666.6667 East Timor,TLS,2004,714.2857 East Timor,TLS,2005,714.2857 East Timor,TLS,2006,714.2857 East Timor,TLS,2007,666.6666 East Timor,TLS,2008,636.36365 -East Timor,TLS,2009,692.3078 +East Timor,TLS,2009,692.30774 East Timor,TLS,2010,642.8572 East Timor,TLS,2011,642.8572 -East Timor,TLS,2012,692.3078 -East Timor,TLS,2013,655.17236 +East Timor,TLS,2012,692.30774 +East Timor,TLS,2013,655.1724 East Timor,TLS,2014,657.1429 East Timor,TLS,2015,657.1429 East Timor,TLS,2016,658.5366 @@ -1490,42 +1504,44 @@ East Timor,TLS,2019,653.8462 East Timor,TLS,2020,660 East Timor,TLS,2021,666.6667 East Timor,TLS,2022,666.6667 -Ecuador,ECU,2000,204.17854 -Ecuador,ECU,2001,254.09836 -Ecuador,ECU,2002,262.48938 +East Timor,TLS,2023,666.6667 +Ecuador,ECU,2000,204.17856 +Ecuador,ECU,2001,254.09837 +Ecuador,ECU,2002,262.4894 Ecuador,ECU,2003,274.39026 Ecuador,ECU,2004,303.90625 -Ecuador,ECU,2005,327.53165 -Ecuador,ECU,2006,352.2727 -Ecuador,ECU,2007,340.28983 +Ecuador,ECU,2005,327.53162 +Ecuador,ECU,2006,352.27274 +Ecuador,ECU,2007,340.28986 Ecuador,ECU,2008,283.93726 -Ecuador,ECU,2009,349.66962 +Ecuador,ECU,2009,349.6696 Ecuador,ECU,2010,389.80432 -Ecuador,ECU,2011,325.50174 +Ecuador,ECU,2011,325.5017 Ecuador,ECU,2012,329.96042 Ecuador,ECU,2013,368.7392 -Ecuador,ECU,2014,369.1608 +Ecuador,ECU,2014,369.16083 Ecuador,ECU,2015,345.72205 Ecuador,ECU,2016,288.6598 Ecuador,ECU,2017,201.2217 -Ecuador,ECU,2018,206.61157 -Ecuador,ECU,2019,169.29742 -Ecuador,ECU,2020,159.47585 -Ecuador,ECU,2021,149.12828 -Ecuador,ECU,2022,150.68912 -Ecuador,ECU,2023,150.22354 +Ecuador,ECU,2018,210.05815 +Ecuador,ECU,2019,171.95685 +Ecuador,ECU,2020,159.47586 +Ecuador,ECU,2021,149.12827 +Ecuador,ECU,2022,181.18571 +Ecuador,ECU,2023,176.2811 +Ecuador,ECU,2024,209.69698 Egypt,EGY,2000,525.64105 Egypt,EGY,2001,533.7629 -Egypt,EGY,2002,542.9665 +Egypt,EGY,2002,542.96643 Egypt,EGY,2003,556.0256 -Egypt,EGY,2004,559.89825 -Egypt,EGY,2005,566.9906 -Egypt,EGY,2006,570.2131 +Egypt,EGY,2004,559.8982 +Egypt,EGY,2005,566.99066 +Egypt,EGY,2006,570.21313 Egypt,EGY,2007,567.57886 Egypt,EGY,2008,567.3913 Egypt,EGY,2009,576.45044 Egypt,EGY,2010,582.70447 -Egypt,EGY,2011,583.1773 +Egypt,EGY,2011,583.17737 Egypt,EGY,2012,587.8777 Egypt,EGY,2013,589.1689 Egypt,EGY,2014,591.8594 @@ -1533,35 +1549,37 @@ Egypt,EGY,2015,592.25653 Egypt,EGY,2016,592.96454 Egypt,EGY,2017,594.6897 Egypt,EGY,2018,592.40686 -Egypt,EGY,2019,579.42755 -Egypt,EGY,2020,566.4653 -Egypt,EGY,2021,570.3792 -Egypt,EGY,2022,570.12695 -Egypt,EGY,2023,570.30554 +Egypt,EGY,2019,579.44855 +Egypt,EGY,2020,566.33606 +Egypt,EGY,2021,571.3059 +Egypt,EGY,2022,574.7312 +Egypt,EGY,2023,574.0438 +Egypt,EGY,2024,571.9383 El Salvador,SLV,2000,285.71426 El Salvador,SLV,2001,297.7099 -El Salvador,SLV,2002,314.76993 +El Salvador,SLV,2002,314.76996 El Salvador,SLV,2003,283.44672 El Salvador,SLV,2004,303.57144 El Salvador,SLV,2005,284.8233 -El Salvador,SLV,2006,293.80533 -El Salvador,SLV,2007,308.6207 +El Salvador,SLV,2006,293.8053 +El Salvador,SLV,2007,308.62067 El Salvador,SLV,2008,271.81207 El Salvador,SLV,2009,313.14877 -El Salvador,SLV,2010,259.63147 +El Salvador,SLV,2010,259.6315 El Salvador,SLV,2011,273.0263 El Salvador,SLV,2012,290.5297 El Salvador,SLV,2013,291.40128 El Salvador,SLV,2014,289.80893 El Salvador,SLV,2015,307.5658 -El Salvador,SLV,2016,308.19675 +El Salvador,SLV,2016,308.19672 El Salvador,SLV,2017,211.46953 -El Salvador,SLV,2018,195.57193 -El Salvador,SLV,2019,239.73726 +El Salvador,SLV,2018,196.66048 +El Salvador,SLV,2019,240.13159 El Salvador,SLV,2020,153.23856 -El Salvador,SLV,2021,155.9055 -El Salvador,SLV,2022,116.53543 -El Salvador,SLV,2023,271.46817 +El Salvador,SLV,2021,152.77779 +El Salvador,SLV,2022,112.95181 +El Salvador,SLV,2023,118.46153 +El Salvador,SLV,2024,101.56249 Equatorial Guinea,GNQ,2000,800 Equatorial Guinea,GNQ,2001,666.6667 Equatorial Guinea,GNQ,2002,714.2857 @@ -1570,10 +1588,10 @@ Equatorial Guinea,GNQ,2004,777.7778 Equatorial Guinea,GNQ,2005,800 Equatorial Guinea,GNQ,2006,818.1819 Equatorial Guinea,GNQ,2007,846.1539 -Equatorial Guinea,GNQ,2008,650 +Equatorial Guinea,GNQ,2008,649.99994 Equatorial Guinea,GNQ,2009,681.81824 Equatorial Guinea,GNQ,2010,772.7273 -Equatorial Guinea,GNQ,2011,800 +Equatorial Guinea,GNQ,2011,799.99994 Equatorial Guinea,GNQ,2012,740.7407 Equatorial Guinea,GNQ,2013,623.52936 Equatorial Guinea,GNQ,2014,577.9816 @@ -1581,69 +1599,62 @@ Equatorial Guinea,GNQ,2015,569.2308 Equatorial Guinea,GNQ,2016,585.7143 Equatorial Guinea,GNQ,2017,597.0149 Equatorial Guinea,GNQ,2018,592.5926 -Equatorial Guinea,GNQ,2019,599.99994 +Equatorial Guinea,GNQ,2019,600 Equatorial Guinea,GNQ,2020,574.4681 -Equatorial Guinea,GNQ,2021,594.4056 -Equatorial Guinea,GNQ,2022,591.83673 +Equatorial Guinea,GNQ,2021,594.40564 +Equatorial Guinea,GNQ,2022,602.64905 +Equatorial Guinea,GNQ,2023,605.0955 Eritrea,ERI,2000,666.6667 Eritrea,ERI,2001,652.17395 Eritrea,ERI,2002,653.8462 Eritrea,ERI,2003,642.8572 Eritrea,ERI,2004,642.8572 -Eritrea,ERI,2005,655.17236 -Eritrea,ERI,2006,666.66675 -Eritrea,ERI,2007,655.17236 +Eritrea,ERI,2005,655.1724 +Eritrea,ERI,2006,666.6667 +Eritrea,ERI,2007,655.1724 Eritrea,ERI,2008,642.8572 -Eritrea,ERI,2009,655.17236 -Eritrea,ERI,2010,645.16125 -Eritrea,ERI,2011,656.24994 -Eritrea,ERI,2012,656.24994 -Eritrea,ERI,2013,666.66675 -Eritrea,ERI,2014,666.66675 +Eritrea,ERI,2009,655.1724 +Eritrea,ERI,2010,645.1613 +Eritrea,ERI,2011,656.25 +Eritrea,ERI,2012,656.25 +Eritrea,ERI,2013,666.6667 +Eritrea,ERI,2014,666.6667 Eritrea,ERI,2015,653.8462 -Eritrea,ERI,2016,666.66675 -Eritrea,ERI,2017,647.0588 -Eritrea,ERI,2018,628.5714 -Eritrea,ERI,2019,638.8889 -Eritrea,ERI,2020,648.6486 -Eritrea,ERI,2021,631.5789 -Eritrea,ERI,2022,631.5789 -Estonia,EST,1990,639.69727 -Estonia,EST,1991,636.36365 -Estonia,EST,1992,650.0423 -Estonia,EST,1993,653.5088 -Estonia,EST,1994,650.2732 -Estonia,EST,1995,647.87115 -Estonia,EST,1996,647.9648 -Estonia,EST,1997,648.58997 -Estonia,EST,1998,646.7836 -Estonia,EST,1999,648.12573 +Eritrea,ERI,2016,666.6667 +Eritrea,ERI,2017,647.05884 +Eritrea,ERI,2018,564.1026 +Eritrea,ERI,2019,575 +Eritrea,ERI,2020,585.36584 +Eritrea,ERI,2021,571.4286 +Eritrea,ERI,2022,590.90906 +Eritrea,ERI,2023,590.90906 Estonia,EST,2000,634.5476 Estonia,EST,2001,633.6867 Estonia,EST,2002,634.2321 -Estonia,EST,2003,638.7795 +Estonia,EST,2003,638.77954 Estonia,EST,2004,639.18524 Estonia,EST,2005,634.3137 Estonia,EST,2006,632.0658 Estonia,EST,2007,642.03613 Estonia,EST,2008,633.2703 -Estonia,EST,2009,621.868 +Estonia,EST,2009,621.8679 Estonia,EST,2010,609.8689 Estonia,EST,2011,604.3444 -Estonia,EST,2012,592.8093 +Estonia,EST,2012,592.8094 Estonia,EST,2013,606.17474 Estonia,EST,2014,595.98395 Estonia,EST,2015,573.39905 -Estonia,EST,2016,588.3319 +Estonia,EST,2016,588.332 Estonia,EST,2017,582.8268 Estonia,EST,2018,573.9693 -Estonia,EST,2019,509.85544 -Estonia,EST,2020,404.60526 +Estonia,EST,2019,509.85547 +Estonia,EST,2020,404.6053 Estonia,EST,2021,450.62585 -Estonia,EST,2022,489.8877 -Estonia,EST,2023,416.6667 +Estonia,EST,2022,487.6958 +Estonia,EST,2023,388.03418 +Estonia,EST,2024,341.0214 Eswatini,SWZ,2000,166.66666 -Eswatini,SWZ,2001,171.42856 +Eswatini,SWZ,2001,171.42857 Eswatini,SWZ,2002,194.44444 Eswatini,SWZ,2003,218.75 Eswatini,SWZ,2004,224.13792 @@ -1654,110 +1665,115 @@ Eswatini,SWZ,2008,188.40579 Eswatini,SWZ,2009,181.81818 Eswatini,SWZ,2010,175.67567 Eswatini,SWZ,2011,155.17242 -Eswatini,SWZ,2012,173.07695 -Eswatini,SWZ,2013,150.00002 +Eswatini,SWZ,2012,173.07693 +Eswatini,SWZ,2013,150 Eswatini,SWZ,2014,166.66667 Eswatini,SWZ,2015,184.21053 -Eswatini,SWZ,2016,222.22223 +Eswatini,SWZ,2016,222.22221 Eswatini,SWZ,2017,139.53488 Eswatini,SWZ,2018,142.85715 Eswatini,SWZ,2019,204.08163 -Eswatini,SWZ,2020,188.67926 -Eswatini,SWZ,2021,175.4386 -Eswatini,SWZ,2022,172.41379 +Eswatini,SWZ,2020,163.2653 +Eswatini,SWZ,2021,137.2549 +Eswatini,SWZ,2022,127.27273 +Eswatini,SWZ,2023,142.85713 Ethiopia,ETH,2000,35.502956 Ethiopia,ETH,2001,34.313725 Ethiopia,ETH,2002,33.980583 -Ethiopia,ETH,2003,30.042921 -Ethiopia,ETH,2004,31.128407 -Ethiopia,ETH,2005,31.250002 +Ethiopia,ETH,2003,30.04292 +Ethiopia,ETH,2004,31.128405 +Ethiopia,ETH,2005,31.25 Ethiopia,ETH,2006,30.211481 Ethiopia,ETH,2007,34.98542 Ethiopia,ETH,2008,30.030031 Ethiopia,ETH,2009,35.911602 Ethiopia,ETH,2010,31.936125 -Ethiopia,ETH,2011,29.96845 -Ethiopia,ETH,2012,25.099073 +Ethiopia,ETH,2011,29.968452 +Ethiopia,ETH,2012,25.099075 Ethiopia,ETH,2013,25.374855 Ethiopia,ETH,2014,23.231256 -Ethiopia,ETH,2015,24.061596 +Ethiopia,ETH,2015,24.061598 Ethiopia,ETH,2016,24.05774 -Ethiopia,ETH,2017,23.628048 -Ethiopia,ETH,2018,23.367697 -Ethiopia,ETH,2019,24.55209 -Ethiopia,ETH,2020,24.005053 -Ethiopia,ETH,2021,24.453024 -Ethiopia,ETH,2022,24.643318 -Europe,,2000,421.95215 -Europe,,2001,416.4603 -Europe,,2002,423.0578 -Europe,,2003,430.4509 -Europe,,2004,418.55392 -Europe,,2005,416.35852 -Europe,,2006,421.31198 -Europe,,2007,422.28552 -Europe,,2008,412.82983 -Europe,,2009,397.94214 -Europe,,2010,392.398 -Europe,,2011,398.60916 -Europe,,2012,396.77585 -Europe,,2013,383.72284 -Europe,,2014,369.28345 -Europe,,2015,365.96725 -Europe,,2016,358.07874 -Europe,,2017,352.87637 -Europe,,2018,339.59708 -Europe,,2019,319.25717 -Europe,,2020,295.83896 -Europe,,2021,303.8504 -Europe,,2022,318.78186 -Europe (Ember),,2000,425.22638 -Europe (Ember),,2001,420.56326 -Europe (Ember),,2002,425.3266 -Europe (Ember),,2003,432.3276 -Europe (Ember),,2004,420.00647 -Europe (Ember),,2005,419.4962 -Europe (Ember),,2006,424.28497 -Europe (Ember),,2007,426.8506 -Europe (Ember),,2008,418.46378 -Europe (Ember),,2009,403.73813 -Europe (Ember),,2010,396.60007 -Europe (Ember),,2011,403.5903 -Europe (Ember),,2012,401.4318 -Europe (Ember),,2013,388.2468 -Europe (Ember),,2014,377.53418 -Europe (Ember),,2015,371.58047 -Europe (Ember),,2016,365.22403 -Europe (Ember),,2017,361.4878 -Europe (Ember),,2018,349.48416 -Europe (Ember),,2019,327.50473 -Europe (Ember),,2020,305.47232 -Europe (Ember),,2021,314.40738 -Europe (Ember),,2022,327.6894 -Europe (Ember),,2023,300.23145 -European Union (27),,2000,419.09564 -European Union (27),,2001,409.16153 +Ethiopia,ETH,2017,23.62805 +Ethiopia,ETH,2018,23.893185 +Ethiopia,ETH,2019,24.167212 +Ethiopia,ETH,2020,24.327785 +Ethiopia,ETH,2021,22.81369 +Ethiopia,ETH,2022,23.549685 +Ethiopia,ETH,2023,23.54874 +Europe,,2000,422.01865 +Europe,,2001,416.54684 +Europe,,2002,423.12045 +Europe,,2003,430.49982 +Europe,,2004,418.61597 +Europe,,2005,416.42578 +Europe,,2006,421.38284 +Europe,,2007,422.3521 +Europe,,2008,412.89026 +Europe,,2009,398.01028 +Europe,,2010,392.4654 +Europe,,2011,398.68347 +Europe,,2012,396.84482 +Europe,,2013,383.79022 +Europe,,2014,369.34964 +Europe,,2015,366.03732 +Europe,,2016,358.15335 +Europe,,2017,352.9357 +Europe,,2018,339.71255 +Europe,,2019,319.489 +Europe,,2020,296.0686 +Europe,,2021,304.3741 +Europe,,2022,318.9479 +Europe,,2023,281.0799 +Europe (Ember),,2000,426.24332 +Europe (Ember),,2001,421.61392 +Europe (Ember),,2002,426.28244 +Europe (Ember),,2003,433.309 +Europe (Ember),,2004,420.97736 +Europe (Ember),,2005,420.63364 +Europe (Ember),,2006,425.48398 +Europe (Ember),,2007,428.21906 +Europe (Ember),,2008,419.89764 +Europe (Ember),,2009,405.18427 +Europe (Ember),,2010,397.97064 +Europe (Ember),,2011,405.1195 +Europe (Ember),,2012,402.9738 +Europe (Ember),,2013,389.76105 +Europe (Ember),,2014,379.32324 +Europe (Ember),,2015,373.22894 +Europe (Ember),,2016,366.99945 +Europe (Ember),,2017,363.42276 +Europe (Ember),,2018,351.45807 +Europe (Ember),,2019,329.40863 +Europe (Ember),,2020,307.4136 +Europe (Ember),,2021,316.76807 +Europe (Ember),,2022,329.6908 +Europe (Ember),,2023,300.27658 +Europe (Ember),,2024,284.16852 +European Union (27),,2000,419.0956 +European Union (27),,2001,409.1615 European Union (27),,2002,419.65817 -European Union (27),,2003,424.24066 +European Union (27),,2003,424.2407 European Union (27),,2004,412.04453 European Union (27),,2005,409.94046 -European Union (27),,2006,408.72287 +European Union (27),,2006,408.7229 European Union (27),,2007,413.10452 European Union (27),,2008,392.42636 European Union (27),,2009,382.45203 -European Union (27),,2010,369.4058 -European Union (27),,2011,376.7041 +European Union (27),,2010,369.40326 +European Union (27),,2011,376.72073 European Union (27),,2012,371.94797 -European Union (27),,2013,357.16818 -European Union (27),,2014,342.93982 -European Union (27),,2015,348.38794 -European Union (27),,2016,341.45203 -European Union (27),,2017,340.37274 -European Union (27),,2018,321.31436 -European Union (27),,2019,287.35977 -European Union (27),,2020,261.25558 -European Union (27),,2021,272.628 -European Union (27),,2022,291.85495 +European Union (27),,2013,357.16693 +European Union (27),,2014,342.9456 +European Union (27),,2015,348.39474 +European Union (27),,2016,341.45538 +European Union (27),,2017,340.3681 +European Union (27),,2018,321.305 +European Union (27),,2019,287.3449 +European Union (27),,2020,261.23105 +European Union (27),,2021,272.61124 +European Union (27),,2022,291.70627 +European Union (27),,2023,237.48735 Falkland Islands,FLK,2000,1000 Falkland Islands,FLK,2001,1000 Falkland Islands,FLK,2002,1000 @@ -1775,81 +1791,72 @@ Falkland Islands,FLK,2013,500 Falkland Islands,FLK,2014,500 Falkland Islands,FLK,2015,500 Falkland Islands,FLK,2016,500 -Falkland Islands,FLK,2017,500 -Falkland Islands,FLK,2018,500 -Falkland Islands,FLK,2019,500 -Falkland Islands,FLK,2020,500 -Falkland Islands,FLK,2021,500 -Falkland Islands,FLK,2022,500 +Falkland Islands,FLK,2017,1000 +Falkland Islands,FLK,2018,1000 +Falkland Islands,FLK,2019,1000 +Falkland Islands,FLK,2020,1000 +Falkland Islands,FLK,2021,1000 +Falkland Islands,FLK,2022,1000 Faroe Islands,FRO,2000,409.09094 -Faroe Islands,FRO,2001,458.3333 +Faroe Islands,FRO,2001,458.33334 Faroe Islands,FRO,2002,375.00003 -Faroe Islands,FRO,2003,458.3333 +Faroe Islands,FRO,2003,458.33334 Faroe Islands,FRO,2004,400 -Faroe Islands,FRO,2005,360.00003 -Faroe Islands,FRO,2006,360.00003 +Faroe Islands,FRO,2005,360 +Faroe Islands,FRO,2006,360 Faroe Islands,FRO,2007,370.37036 -Faroe Islands,FRO,2008,407.4074 -Faroe Islands,FRO,2009,407.4074 +Faroe Islands,FRO,2008,407.40738 +Faroe Islands,FRO,2009,407.40738 Faroe Islands,FRO,2010,464.2857 -Faroe Islands,FRO,2011,407.4074 -Faroe Islands,FRO,2012,413.7931 -Faroe Islands,FRO,2013,413.7931 -Faroe Islands,FRO,2014,322.58063 +Faroe Islands,FRO,2011,407.40738 +Faroe Islands,FRO,2012,413.79312 +Faroe Islands,FRO,2013,413.79312 +Faroe Islands,FRO,2014,322.58066 Faroe Islands,FRO,2015,281.25003 Faroe Islands,FRO,2016,354.8387 Faroe Islands,FRO,2017,333.3333 -Faroe Islands,FRO,2018,342.85712 +Faroe Islands,FRO,2018,342.85715 Faroe Islands,FRO,2019,394.73688 Faroe Islands,FRO,2020,390.2439 -Faroe Islands,FRO,2021,404.76193 -Faroe Islands,FRO,2022,404.76193 +Faroe Islands,FRO,2021,395.34885 +Faroe Islands,FRO,2022,354.1667 Fiji,FJI,2000,240.00002 -Fiji,FJI,2001,215.18988 -Fiji,FJI,2002,240.96385 +Fiji,FJI,2001,215.18987 +Fiji,FJI,2002,240.96387 Fiji,FJI,2003,344.8276 Fiji,FJI,2004,329.54544 Fiji,FJI,2005,359.55057 Fiji,FJI,2006,355.55554 Fiji,FJI,2007,235.95505 -Fiji,FJI,2008,238.09523 +Fiji,FJI,2008,238.09525 Fiji,FJI,2009,265.06024 Fiji,FJI,2010,340.90912 -Fiji,FJI,2011,305.88235 +Fiji,FJI,2011,305.88232 Fiji,FJI,2012,244.18604 Fiji,FJI,2013,261.36365 Fiji,FJI,2014,362.63736 Fiji,FJI,2015,365.5914 -Fiji,FJI,2016,305.26312 -Fiji,FJI,2017,343.13724 -Fiji,FJI,2018,298.07693 -Fiji,FJI,2019,309.0909 -Fiji,FJI,2020,274.5098 -Fiji,FJI,2021,277.22772 -Fiji,FJI,2022,288.46158 -Finland,FIN,1990,281.6409 -Finland,FIN,1991,276.225 -Finland,FIN,1992,244.54071 -Finland,FIN,1993,278.2509 -Finland,FIN,1994,336.17798 -Finland,FIN,1995,303.75003 -Finland,FIN,1996,350.5988 -Finland,FIN,1997,318.4778 -Finland,FIN,1998,250.17828 -Finland,FIN,1999,262.7485 +Fiji,FJI,2016,305.26315 +Fiji,FJI,2017,343.13727 +Fiji,FJI,2018,300 +Fiji,FJI,2019,307.01755 +Fiji,FJI,2020,268.5185 +Fiji,FJI,2021,273.5849 +Fiji,FJI,2022,280.37384 +Fiji,FJI,2023,278.26086 Finland,FIN,2000,246.67714 -Finland,FIN,2001,284.7549 -Finland,FIN,2002,316.07047 -Finland,FIN,2003,371.26245 -Finland,FIN,2004,325.2563 -Finland,FIN,2005,230.72563 -Finland,FIN,2006,335.64227 +Finland,FIN,2001,284.75488 +Finland,FIN,2002,316.0705 +Finland,FIN,2003,371.26248 +Finland,FIN,2004,325.25632 +Finland,FIN,2005,230.72565 +Finland,FIN,2006,335.64224 Finland,FIN,2007,308.98117 -Finland,FIN,2008,246.15388 +Finland,FIN,2008,246.15387 Finland,FIN,2009,276.02 Finland,FIN,2010,314.73215 Finland,FIN,2011,268.16324 -Finland,FIN,2012,215.16827 +Finland,FIN,2012,215.16829 Finland,FIN,2013,257.50772 Finland,FIN,2014,226.28488 Finland,FIN,2015,181.89767 @@ -1859,42 +1866,34 @@ Finland,FIN,2018,192.93748 Finland,FIN,2019,171.11208 Finland,FIN,2020,130.1271 Finland,FIN,2021,135.48746 -Finland,FIN,2022,130.23512 -Finland,FIN,2023,79.158325 -France,FRA,1990,104.24478 -France,FRA,1991,121.68584 -France,FRA,1992,101.888596 -France,FRA,1993,68.750404 -France,FRA,1994,67.27434 -France,FRA,1995,73.455795 -France,FRA,1996,79.22566 -France,FRA,1997,70.85779 -France,FRA,1998,96.752525 -France,FRA,1999,87.365746 +Finland,FIN,2022,129.83234 +Finland,FIN,2023,81.43962 +Finland,FIN,2024,72.2488 France,FRA,2000,79.76566 France,FRA,2001,68.632515 -France,FRA,2002,75.21224 +France,FRA,2002,75.21225 France,FRA,2003,80.85776 -France,FRA,2004,77.18524 +France,FRA,2004,77.185234 France,FRA,2005,85.532555 France,FRA,2006,76.57324 France,FRA,2007,80.11177 France,FRA,2008,76.03443 France,FRA,2009,75.59514 -France,FRA,2010,78.56178 -France,FRA,2011,74.38934 +France,FRA,2010,78.561775 +France,FRA,2011,74.62344 France,FRA,2012,77.59258 -France,FRA,2013,76.273544 -France,FRA,2014,52.14827 -France,FRA,2015,59.117588 -France,FRA,2016,68.557625 -France,FRA,2017,78.2143 -France,FRA,2018,59.937843 -France,FRA,2019,59.834198 -France,FRA,2020,59.083496 -France,FRA,2021,59.76878 -France,FRA,2022,78.81113 -France,FRA,2023,56.03859 +France,FRA,2013,76.27222 +France,FRA,2014,52.19756 +France,FRA,2015,59.149334 +France,FRA,2016,68.6064 +France,FRA,2017,78.26258 +France,FRA,2018,59.92359 +France,FRA,2019,59.78751 +France,FRA,2020,59.08005 +France,FRA,2021,59.787445 +France,FRA,2022,78.55024 +France,FRA,2023,55.27541 +France,FRA,2024,44.179085 French Guiana,GUF,2000,86.95652 French Guiana,GUF,2001,216.66666 French Guiana,GUF,2002,238.09525 @@ -1904,119 +1903,123 @@ French Guiana,GUF,2005,277.77777 French Guiana,GUF,2006,186.66667 French Guiana,GUF,2007,229.72974 French Guiana,GUF,2008,233.76625 -French Guiana,GUF,2009,382.71603 +French Guiana,GUF,2009,382.71606 French Guiana,GUF,2010,294.11765 French Guiana,GUF,2011,267.44186 -French Guiana,GUF,2012,211.1111 +French Guiana,GUF,2012,211.11111 French Guiana,GUF,2013,277.77777 French Guiana,GUF,2014,268.8172 French Guiana,GUF,2015,284.21054 French Guiana,GUF,2016,311.82794 -French Guiana,GUF,2017,223.40424 -French Guiana,GUF,2018,244.68085 -French Guiana,GUF,2019,343.75 -French Guiana,GUF,2020,323.23233 +French Guiana,GUF,2017,223.40425 +French Guiana,GUF,2018,244.68086 +French Guiana,GUF,2019,343.75003 +French Guiana,GUF,2020,323.2323 French Guiana,GUF,2021,217.82178 -French Polynesia,PYF,2000,463.4146 -French Polynesia,PYF,2001,499.99997 +French Guiana,GUF,2022,204.08163 +French Polynesia,PYF,2000,463.41464 +French Polynesia,PYF,2001,500 French Polynesia,PYF,2002,478.26086 -French Polynesia,PYF,2003,446.80847 +French Polynesia,PYF,2003,446.8085 French Polynesia,PYF,2004,444.44443 -French Polynesia,PYF,2005,431.37253 +French Polynesia,PYF,2005,431.37256 French Polynesia,PYF,2006,415.09436 French Polynesia,PYF,2007,418.18182 French Polynesia,PYF,2008,421.0526 -French Polynesia,PYF,2009,413.7931 +French Polynesia,PYF,2009,413.79312 French Polynesia,PYF,2010,443.03796 French Polynesia,PYF,2011,472.97296 -French Polynesia,PYF,2012,472.22223 -French Polynesia,PYF,2013,500.00003 +French Polynesia,PYF,2012,472.2222 +French Polynesia,PYF,2013,500 French Polynesia,PYF,2014,470.58823 French Polynesia,PYF,2015,470.58823 French Polynesia,PYF,2016,455.88235 French Polynesia,PYF,2017,449.27536 French Polynesia,PYF,2018,470.58823 -French Polynesia,PYF,2019,450.70422 -French Polynesia,PYF,2020,441.17648 -French Polynesia,PYF,2021,442.85715 -French Polynesia,PYF,2022,442.85715 -G20 (Ember),,2000,525.8432 -G20 (Ember),,2001,526.3674 -G20 (Ember),,2002,530.31573 -G20 (Ember),,2003,543.45624 -G20 (Ember),,2004,537.7566 -G20 (Ember),,2005,543.3346 -G20 (Ember),,2006,545.8854 -G20 (Ember),,2007,554.95917 -G20 (Ember),,2008,546.7801 -G20 (Ember),,2009,541.937 -G20 (Ember),,2010,541.8596 -G20 (Ember),,2011,549.68036 -G20 (Ember),,2012,548.0926 -G20 (Ember),,2013,548.3489 -G20 (Ember),,2014,543.3012 -G20 (Ember),,2015,529.4446 -G20 (Ember),,2016,519.9324 -G20 (Ember),,2017,516.9929 -G20 (Ember),,2018,512.67645 -G20 (Ember),,2019,497.34088 -G20 (Ember),,2020,482.80695 -G20 (Ember),,2021,487.81686 -G20 (Ember),,2022,482.9173 -G20 (Ember),,2023,477.06424 -G7 (Ember),,2000,480.75983 -G7 (Ember),,2001,480.8017 -G7 (Ember),,2002,478.7297 -G7 (Ember),,2003,490.2778 -G7 (Ember),,2004,481.35345 -G7 (Ember),,2005,483.09348 -G7 (Ember),,2006,476.2257 -G7 (Ember),,2007,483.7262 -G7 (Ember),,2008,477.3349 -G7 (Ember),,2009,456.39703 -G7 (Ember),,2010,458.68073 -G7 (Ember),,2011,454.39825 -G7 (Ember),,2012,459.35052 -G7 (Ember),,2013,455.35104 -G7 (Ember),,2014,447.28845 -G7 (Ember),,2015,428.3243 -G7 (Ember),,2016,413.6689 -G7 (Ember),,2017,400.6014 -G7 (Ember),,2018,390.88675 -G7 (Ember),,2019,370.66937 -G7 (Ember),,2020,351.91632 -G7 (Ember),,2021,360.7223 -G7 (Ember),,2022,361.52267 -G7 (Ember),,2023,340.91586 -Gabon,GAB,2000,305.3435 +French Polynesia,PYF,2019,463.7681 +French Polynesia,PYF,2020,454.54544 +French Polynesia,PYF,2021,469.69696 +French Polynesia,PYF,2022,436.61972 +G20 (Ember),,2000,536.7785 +G20 (Ember),,2001,537.024 +G20 (Ember),,2002,540.756 +G20 (Ember),,2003,553.7037 +G20 (Ember),,2004,547.69385 +G20 (Ember),,2005,553.17896 +G20 (Ember),,2006,555.2828 +G20 (Ember),,2007,564.2453 +G20 (Ember),,2008,555.7449 +G20 (Ember),,2009,550.25244 +G20 (Ember),,2010,550.0313 +G20 (Ember),,2011,557.29126 +G20 (Ember),,2012,555.2373 +G20 (Ember),,2013,555.3173 +G20 (Ember),,2014,550.1501 +G20 (Ember),,2015,535.89404 +G20 (Ember),,2016,525.947 +G20 (Ember),,2017,522.9976 +G20 (Ember),,2018,518.57574 +G20 (Ember),,2019,502.7335 +G20 (Ember),,2020,487.59927 +G20 (Ember),,2021,492.6144 +G20 (Ember),,2022,487.50784 +G20 (Ember),,2023,481.50943 +G20 (Ember),,2024,468.98517 +G7 (Ember),,2000,499.81528 +G7 (Ember),,2001,499.76727 +G7 (Ember),,2002,497.54184 +G7 (Ember),,2003,509.26245 +G7 (Ember),,2004,500.26212 +G7 (Ember),,2005,502.09006 +G7 (Ember),,2006,494.9029 +G7 (Ember),,2007,502.62518 +G7 (Ember),,2008,495.91113 +G7 (Ember),,2009,474.27917 +G7 (Ember),,2010,476.82785 +G7 (Ember),,2011,471.95102 +G7 (Ember),,2012,476.22208 +G7 (Ember),,2013,472.2594 +G7 (Ember),,2014,464.3185 +G7 (Ember),,2015,444.52466 +G7 (Ember),,2016,429.3176 +G7 (Ember),,2017,416.76355 +G7 (Ember),,2018,407.13922 +G7 (Ember),,2019,386.0507 +G7 (Ember),,2020,365.97696 +G7 (Ember),,2021,375.52185 +G7 (Ember),,2022,375.76334 +G7 (Ember),,2023,353.61453 +G7 (Ember),,2024,342.5886 +Gabon,GAB,2000,305.34354 Gabon,GAB,2001,307.14285 Gabon,GAB,2002,306.12244 Gabon,GAB,2003,326.7974 -Gabon,GAB,2004,333.3333 +Gabon,GAB,2004,333.33334 Gabon,GAB,2005,375.79614 Gabon,GAB,2006,337.3494 Gabon,GAB,2007,404.62427 -Gabon,GAB,2008,403.3149 +Gabon,GAB,2008,403.31494 Gabon,GAB,2009,398.9071 -Gabon,GAB,2010,445.59586 +Gabon,GAB,2010,445.5959 Gabon,GAB,2011,492.61084 -Gabon,GAB,2012,481.48148 +Gabon,GAB,2012,481.48145 Gabon,GAB,2013,454.54547 Gabon,GAB,2014,497.48743 Gabon,GAB,2015,476.41513 -Gabon,GAB,2016,525.1141 +Gabon,GAB,2016,525.11414 Gabon,GAB,2017,495.5357 -Gabon,GAB,2018,475.77097 -Gabon,GAB,2019,484.7162 -Gabon,GAB,2020,491.59662 -Gabon,GAB,2021,482.906 -Gabon,GAB,2022,491.59662 -Gambia,GMB,2000,692.3078 +Gabon,GAB,2018,438.84894 +Gabon,GAB,2019,443.662 +Gabon,GAB,2020,423.2082 +Gabon,GAB,2021,439.18918 +Gabon,GAB,2022,429.46707 +Gabon,GAB,2023,429.46707 +Gambia,GMB,2000,692.30774 Gambia,GMB,2001,666.6666 Gambia,GMB,2002,666.6666 -Gambia,GMB,2003,647.0588 +Gambia,GMB,2003,647.05884 Gambia,GMB,2004,684.2105 -Gambia,GMB,2005,650 +Gambia,GMB,2005,649.99994 Gambia,GMB,2006,636.36365 Gambia,GMB,2007,652.17395 Gambia,GMB,2008,666.6667 @@ -2028,22 +2031,23 @@ Gambia,GMB,2013,640 Gambia,GMB,2014,640 Gambia,GMB,2015,642.8572 Gambia,GMB,2016,666.6666 -Gambia,GMB,2017,645.16125 -Gambia,GMB,2018,666.6666 +Gambia,GMB,2017,645.1613 +Gambia,GMB,2018,666.6667 Gambia,GMB,2019,652.17395 Gambia,GMB,2020,666.6667 Gambia,GMB,2021,660 Gambia,GMB,2022,666.6667 -Georgia,GEO,2000,152.17392 +Gambia,GMB,2023,666.6667 +Georgia,GEO,2000,152.1739 Georgia,GEO,2001,146.80232 Georgia,GEO,2002,65.36857 Georgia,GEO,2003,77.46479 -Georgia,GEO,2004,100.583084 -Georgia,GEO,2005,111.11111 +Georgia,GEO,2004,100.58309 +Georgia,GEO,2005,111.111115 Georgia,GEO,2006,200.80322 Georgia,GEO,2007,148.23529 Georgia,GEO,2008,116.94511 -Georgia,GEO,2009,94.837944 +Georgia,GEO,2009,94.83794 Georgia,GEO,2010,65.32663 Georgia,GEO,2011,157.68463 Georgia,GEO,2012,177.75468 @@ -2051,165 +2055,150 @@ Georgia,GEO,2013,131.26253 Georgia,GEO,2014,143.82896 Georgia,GEO,2015,157.2093 Georgia,GEO,2016,141.86249 -Georgia,GEO,2017,141.60838 -Georgia,GEO,2018,128.73755 +Georgia,GEO,2017,141.6084 +Georgia,GEO,2018,128.73753 Georgia,GEO,2019,167.7909 -Georgia,GEO,2020,175.62726 -Georgia,GEO,2021,136.8671 +Georgia,GEO,2020,175.62724 +Georgia,GEO,2021,136.86708 Georgia,GEO,2022,167.01755 -Georgia,GEO,2023,167.59389 -Germany,DEU,1990,622.1923 -Germany,DEU,1991,628.1855 -Germany,DEU,1992,605.09204 -Germany,DEU,1993,607.7547 -Germany,DEU,1994,604.2798 -Germany,DEU,1995,596.3409 -Germany,DEU,1996,591.303 -Germany,DEU,1997,575.26953 -Germany,DEU,1998,587.008 -Germany,DEU,1999,566.3881 +Georgia,GEO,2023,168.05557 +Georgia,GEO,2024,143.0575 Germany,DEU,2000,573.1259 -Germany,DEU,2001,565.4472 +Germany,DEU,2001,565.44714 Germany,DEU,2002,569.5662 Germany,DEU,2003,569.95496 Germany,DEU,2004,554.6588 -Germany,DEU,2005,546.27124 +Germany,DEU,2005,546.2712 Germany,DEU,2006,536.16315 Germany,DEU,2007,551.78265 Germany,DEU,2008,525.40765 Germany,DEU,2009,521.5491 Germany,DEU,2010,516.637 -Germany,DEU,2011,529.6695 -Germany,DEU,2012,534.16364 +Germany,DEU,2011,529.66943 +Germany,DEU,2012,534.1637 Germany,DEU,2013,538.0267 -Germany,DEU,2014,521.6808 -Germany,DEU,2015,503.59064 +Germany,DEU,2014,521.68085 +Germany,DEU,2015,503.5906 Germany,DEU,2016,498.3009 -Germany,DEU,2017,469.71103 -Germany,DEU,2018,454.54538 +Germany,DEU,2017,469.71106 +Germany,DEU,2018,454.5454 Germany,DEU,2019,392.49112 Germany,DEU,2020,355.6595 -Germany,DEU,2021,394.53833 -Germany,DEU,2022,420.05743 -Germany,DEU,2023,380.95047 +Germany,DEU,2021,394.84824 +Germany,DEU,2022,420.10654 +Germany,DEU,2023,371.31064 +Germany,DEU,2024,344.13992 Ghana,GHA,2000,79.49791 Ghana,GHA,2001,126.76057 -Ghana,GHA,2002,221.45804 -Ghana,GHA,2003,241.4966 -Ghana,GHA,2004,103.33333 +Ghana,GHA,2002,221.45805 +Ghana,GHA,2003,241.49658 +Ghana,GHA,2004,103.333336 Ghana,GHA,2005,132.04749 Ghana,GHA,2006,237.18712 Ghana,GHA,2007,321.83908 -Ghana,GHA,2008,189.90387 +Ghana,GHA,2008,189.90385 Ghana,GHA,2009,173.37808 Ghana,GHA,2010,230.23715 -Ghana,GHA,2011,253.75774 -Ghana,GHA,2012,251.0288 +Ghana,GHA,2011,253.75772 +Ghana,GHA,2012,251.02881 Ghana,GHA,2013,266.35873 Ghana,GHA,2014,269.11316 Ghana,GHA,2015,369.4158 Ghana,GHA,2016,410.0228 -Ghana,GHA,2017,407.51883 +Ghana,GHA,2017,407.5188 Ghana,GHA,2018,421.86453 -Ghana,GHA,2019,416.76306 +Ghana,GHA,2019,416.76303 Ghana,GHA,2020,459.19922 Ghana,GHA,2021,478.2609 -Ghana,GHA,2022,484.00003 -Gibraltar,GIB,2000,692.3078 -Gibraltar,GIB,2001,692.3078 -Gibraltar,GIB,2002,692.3078 -Gibraltar,GIB,2003,692.3078 +Ghana,GHA,2022,469.4229 +Ghana,GHA,2023,452.86124 +Gibraltar,GIB,2000,692.30774 +Gibraltar,GIB,2001,692.30774 +Gibraltar,GIB,2002,692.30774 +Gibraltar,GIB,2003,692.30774 Gibraltar,GIB,2004,642.8572 Gibraltar,GIB,2005,642.8572 Gibraltar,GIB,2006,666.6666 Gibraltar,GIB,2007,687.5 -Gibraltar,GIB,2008,647.0588 -Gibraltar,GIB,2009,647.0588 +Gibraltar,GIB,2008,647.05884 +Gibraltar,GIB,2009,647.05884 Gibraltar,GIB,2010,666.6666 -Gibraltar,GIB,2011,647.0588 +Gibraltar,GIB,2011,647.05884 Gibraltar,GIB,2012,666.6666 Gibraltar,GIB,2013,684.2105 -Gibraltar,GIB,2014,650 +Gibraltar,GIB,2014,649.99994 Gibraltar,GIB,2015,636.36365 Gibraltar,GIB,2016,666.6667 Gibraltar,GIB,2017,666.6667 Gibraltar,GIB,2018,636.36365 Gibraltar,GIB,2019,636.36365 -Gibraltar,GIB,2020,571.4285 +Gibraltar,GIB,2020,571.4286 Gibraltar,GIB,2021,565.21735 -Gibraltar,GIB,2022,600.00006 -Greece,GRC,1990,884.21045 -Greece,GRC,1991,838.5906 -Greece,GRC,1992,870.5115 -Greece,GRC,1993,877.02094 -Greece,GRC,1994,875.8121 -Greece,GRC,1995,848.8031 -Greece,GRC,1996,835.191 -Greece,GRC,1997,850.5022 -Greece,GRC,1998,850.76587 -Greece,GRC,1999,819.79956 +Gibraltar,GIB,2022,571.4286 +Gibraltar,GIB,2023,590.90906 Greece,GRC,2000,822.81836 -Greece,GRC,2001,845.64136 +Greece,GRC,2001,845.6414 Greece,GRC,2002,827.5862 -Greece,GRC,2003,790.3282 +Greece,GRC,2003,790.32825 Greece,GRC,2004,786.15594 Greece,GRC,2005,782.5643 -Greece,GRC,2006,743.141 -Greece,GRC,2007,776.9231 +Greece,GRC,2006,743.1409 +Greece,GRC,2007,776.92303 Greece,GRC,2008,758.92566 Greece,GRC,2009,747.0998 Greece,GRC,2010,707.29553 Greece,GRC,2011,725.06415 Greece,GRC,2012,704.0561 -Greece,GRC,2013,636.6862 +Greece,GRC,2013,636.68616 Greece,GRC,2014,667.87585 Greece,GRC,2015,603.52594 Greece,GRC,2016,571.5082 Greece,GRC,2017,579.74457 -Greece,GRC,2018,545.14355 -Greece,GRC,2019,496.97598 +Greece,GRC,2018,545.1435 +Greece,GRC,2019,496.976 Greece,GRC,2020,423.06885 -Greece,GRC,2021,383.11206 -Greece,GRC,2022,376.6712 -Greece,GRC,2023,336.57352 +Greece,GRC,2021,383.1121 +Greece,GRC,2022,376.8144 +Greece,GRC,2023,337.08783 +Greece,GRC,2024,319.76236 Greenland,GRL,2000,275.86206 -Greenland,GRL,2001,285.7143 +Greenland,GRL,2001,285.71426 Greenland,GRL,2002,290.3226 Greenland,GRL,2003,290.3226 -Greenland,GRL,2004,272.7273 +Greenland,GRL,2004,272.72726 Greenland,GRL,2005,264.7059 Greenland,GRL,2006,285.7143 Greenland,GRL,2007,305.55554 Greenland,GRL,2008,297.2973 -Greenland,GRL,2009,268.29266 +Greenland,GRL,2009,268.2927 Greenland,GRL,2010,219.5122 Greenland,GRL,2011,177.77777 Greenland,GRL,2012,177.77777 -Greenland,GRL,2013,152.17392 -Greenland,GRL,2014,145.83333 +Greenland,GRL,2013,152.1739 +Greenland,GRL,2014,145.83334 Greenland,GRL,2015,137.2549 -Greenland,GRL,2016,124.99999 -Greenland,GRL,2017,180.00002 -Greenland,GRL,2018,185.18518 -Greenland,GRL,2019,188.67926 -Greenland,GRL,2020,158.73015 -Greenland,GRL,2021,158.73015 -Greenland,GRL,2022,178.57143 -Grenada,GRD,2000,692.3078 +Greenland,GRL,2016,125 +Greenland,GRL,2017,180 +Greenland,GRL,2018,153.84616 +Greenland,GRL,2019,122.448975 +Greenland,GRL,2020,134.61539 +Greenland,GRL,2021,117.64706 +Greenland,GRL,2022,111.11111 +Grenada,GRD,2000,692.30774 Grenada,GRD,2001,642.8572 Grenada,GRD,2002,666.6666 Grenada,GRD,2003,687.5 Grenada,GRD,2004,642.8572 Grenada,GRD,2005,666.6666 -Grenada,GRD,2006,647.0588 +Grenada,GRD,2006,647.05884 Grenada,GRD,2007,666.6666 Grenada,GRD,2008,684.2105 -Grenada,GRD,2009,650 +Grenada,GRD,2009,649.99994 Grenada,GRD,2010,666.6667 -Grenada,GRD,2011,650 -Grenada,GRD,2012,650 -Grenada,GRD,2013,650 -Grenada,GRD,2014,650 +Grenada,GRD,2011,649.99994 +Grenada,GRD,2012,649.99994 +Grenada,GRD,2013,649.99994 +Grenada,GRD,2014,649.99994 Grenada,GRD,2015,666.6667 Grenada,GRD,2016,636.36365 Grenada,GRD,2017,652.17395 @@ -2217,18 +2206,18 @@ Grenada,GRD,2018,652.17395 Grenada,GRD,2019,666.6667 Grenada,GRD,2020,636.36365 Grenada,GRD,2021,652.17395 -Grenada,GRD,2022,640 -Guadeloupe,GLP,2000,672.13116 +Grenada,GRD,2022,666.6667 +Guadeloupe,GLP,2000,672.1311 Guadeloupe,GLP,2001,666.6667 -Guadeloupe,GLP,2002,654.1354 +Guadeloupe,GLP,2002,654.1353 Guadeloupe,GLP,2003,659.4203 -Guadeloupe,GLP,2004,636.36365 +Guadeloupe,GLP,2004,636.3637 Guadeloupe,GLP,2005,620 Guadeloupe,GLP,2006,633.98694 Guadeloupe,GLP,2007,625 Guadeloupe,GLP,2008,621.118 Guadeloupe,GLP,2009,613.49695 -Guadeloupe,GLP,2010,641.6185 +Guadeloupe,GLP,2010,641.61847 Guadeloupe,GLP,2011,648.8096 Guadeloupe,GLP,2012,633.72095 Guadeloupe,GLP,2013,614.94257 @@ -2236,79 +2225,82 @@ Guadeloupe,GLP,2014,603.44824 Guadeloupe,GLP,2015,596.5909 Guadeloupe,GLP,2016,610.1695 Guadeloupe,GLP,2017,596.5909 -Guadeloupe,GLP,2018,584.79535 +Guadeloupe,GLP,2018,584.7953 Guadeloupe,GLP,2019,569.76746 Guadeloupe,GLP,2020,547.05884 Guadeloupe,GLP,2021,500 -Guam,GUM,2000,659.7938 -Guam,GUM,2001,656.4103 +Guadeloupe,GLP,2022,493.90244 +Guam,GUM,2000,659.79376 +Guam,GUM,2001,656.4102 Guam,GUM,2002,656.97675 Guam,GUM,2003,657.30334 Guam,GUM,2004,659.57446 -Guam,GUM,2005,657.8947 +Guam,GUM,2005,657.8948 Guam,GUM,2006,656.08466 Guam,GUM,2007,659.57446 Guam,GUM,2008,659.57446 -Guam,GUM,2009,657.75397 -Guam,GUM,2010,657.75397 -Guam,GUM,2011,657.60876 +Guam,GUM,2009,657.754 +Guam,GUM,2010,657.754 +Guam,GUM,2011,657.6087 Guam,GUM,2012,657.30334 Guam,GUM,2013,655.1724 -Guam,GUM,2014,658.82355 +Guam,GUM,2014,658.8235 Guam,GUM,2015,649.1228 Guam,GUM,2016,632.1839 Guam,GUM,2017,638.4181 -Guam,GUM,2018,630.05786 -Guam,GUM,2019,632.1839 -Guam,GUM,2020,629.4118 -Guam,GUM,2021,626.4368 -Guam,GUM,2022,622.8572 +Guam,GUM,2018,630.0578 +Guam,GUM,2019,628.5715 +Guam,GUM,2020,625.731 +Guam,GUM,2021,626.43677 +Guam,GUM,2022,611.11115 Guatemala,GTM,2000,374.613 Guatemala,GTM,2001,426.44873 -Guatemala,GTM,2002,454.955 -Guatemala,GTM,2003,393.63245 +Guatemala,GTM,2002,454.95496 +Guatemala,GTM,2003,393.63242 Guatemala,GTM,2004,389.40808 Guatemala,GTM,2005,358.2734 -Guatemala,GTM,2006,368.4879 +Guatemala,GTM,2006,368.48795 Guatemala,GTM,2007,389.90826 Guatemala,GTM,2008,355.99075 Guatemala,GTM,2009,405.16272 Guatemala,GTM,2010,316.74207 -Guatemala,GTM,2011,315.06845 -Guatemala,GTM,2012,295.94016 +Guatemala,GTM,2011,315.06848 +Guatemala,GTM,2012,295.9402 Guatemala,GTM,2013,301.3834 Guatemala,GTM,2014,300.75186 -Guatemala,GTM,2015,368.8451 +Guatemala,GTM,2015,368.84512 Guatemala,GTM,2016,371.8381 -Guatemala,GTM,2017,312.8039 -Guatemala,GTM,2018,336.49634 -Guatemala,GTM,2019,356.50226 -Guatemala,GTM,2020,264.40408 -Guatemala,GTM,2021,323.96564 -Guatemala,GTM,2022,328.26752 +Guatemala,GTM,2017,312.80386 +Guatemala,GTM,2018,340.37738 +Guatemala,GTM,2019,363.7795 +Guatemala,GTM,2020,265.70047 +Guatemala,GTM,2021,285.7143 +Guatemala,GTM,2022,191.39465 +Guatemala,GTM,2023,272.65973 Guinea,GIN,2000,333.33334 -Guinea,GIN,2001,291.66666 +Guinea,GIN,2001,291.6667 Guinea,GIN,2002,287.1287 Guinea,GIN,2003,360.46512 Guinea,GIN,2004,266.66666 -Guinea,GIN,2005,265.82275 +Guinea,GIN,2005,265.82278 Guinea,GIN,2006,260.27396 Guinea,GIN,2007,217.94873 -Guinea,GIN,2008,246.91357 -Guinea,GIN,2009,237.49998 -Guinea,GIN,2010,174.60316 +Guinea,GIN,2008,246.91359 +Guinea,GIN,2009,237.5 +Guinea,GIN,2010,174.60318 Guinea,GIN,2011,181.81818 -Guinea,GIN,2012,232.87672 -Guinea,GIN,2013,184.61537 -Guinea,GIN,2014,281.69012 +Guinea,GIN,2012,232.87671 +Guinea,GIN,2013,184.61539 +Guinea,GIN,2014,281.69016 Guinea,GIN,2015,157.89474 Guinea,GIN,2016,229.81366 Guinea,GIN,2017,220.33897 -Guinea,GIN,2018,335.57047 -Guinea,GIN,2019,253.65852 -Guinea,GIN,2020,202.31213 -Guinea,GIN,2021,229.99998 -Guinea,GIN,2022,236.84212 +Guinea,GIN,2018,262.8866 +Guinea,GIN,2019,253.65854 +Guinea,GIN,2020,274.19354 +Guinea,GIN,2021,192.30768 +Guinea,GIN,2022,185 +Guinea,GIN,2023,182.71605 Guinea-Bissau,GNB,2000,666.6667 Guinea-Bissau,GNB,2001,500 Guinea-Bissau,GNB,2002,500 @@ -2337,151 +2329,146 @@ Guyana,GUY,2001,652.17395 Guyana,GUY,2002,652.17395 Guyana,GUY,2003,646.34143 Guyana,GUY,2004,651.1628 -Guyana,GUY,2005,646.1538 -Guyana,GUY,2006,651.5152 +Guyana,GUY,2005,646.1539 +Guyana,GUY,2006,651.51514 Guyana,GUY,2007,642.8571 Guyana,GUY,2008,647.8873 -Guyana,GUY,2009,650.6024 +Guyana,GUY,2009,650.6025 Guyana,GUY,2010,655.1724 Guyana,GUY,2011,648.3516 -Guyana,GUY,2012,648.9362 -Guyana,GUY,2013,645.8333 +Guyana,GUY,2012,648.93616 +Guyana,GUY,2013,645.8334 Guyana,GUY,2014,642.8571 -Guyana,GUY,2015,649.99994 +Guyana,GUY,2015,650 Guyana,GUY,2016,648.14813 Guyana,GUY,2017,645.4545 Guyana,GUY,2018,646.0177 Guyana,GUY,2019,640.3509 Guyana,GUY,2020,646.0177 -Guyana,GUY,2021,640.3509 -Guyana,GUY,2022,640.3509 -Haiti,HTI,2000,333.33337 -Haiti,HTI,2001,355.9322 +Guyana,GUY,2021,642.8572 +Guyana,GUY,2022,653.5433 +Guyana,GUY,2023,634.32837 +Haiti,HTI,2000,333.33334 +Haiti,HTI,2001,355.93222 Haiti,HTI,2002,363.63635 Haiti,HTI,2003,358.49057 Haiti,HTI,2004,363.63635 Haiti,HTI,2005,363.63635 -Haiti,HTI,2006,368.42102 +Haiti,HTI,2006,368.42105 Haiti,HTI,2007,434.7826 Haiti,HTI,2008,408.16327 Haiti,HTI,2009,478.87326 -Haiti,HTI,2010,500.00003 +Haiti,HTI,2010,500 Haiti,HTI,2011,575.75757 Haiti,HTI,2012,536.3636 Haiti,HTI,2013,571.42865 -Haiti,HTI,2014,601.9417 -Haiti,HTI,2015,611.65045 -Haiti,HTI,2016,611.11115 -Haiti,HTI,2017,575.4718 -Haiti,HTI,2018,538.4616 +Haiti,HTI,2014,601.9418 +Haiti,HTI,2015,611.6505 +Haiti,HTI,2016,611.1111 +Haiti,HTI,2017,575.47174 +Haiti,HTI,2018,538.46155 Haiti,HTI,2019,556.60376 -Haiti,HTI,2020,575.4718 -Haiti,HTI,2021,572.8155 -Haiti,HTI,2022,567.3077 -High-income countries,,2000,486.49533 -High-income countries,,2001,486.66113 -High-income countries,,2002,486.89227 -High-income countries,,2003,497.4864 -High-income countries,,2004,489.2723 -High-income countries,,2005,489.4379 -High-income countries,,2006,485.97446 -High-income countries,,2007,491.57742 -High-income countries,,2008,483.5415 -High-income countries,,2009,468.80167 -High-income countries,,2010,468.55737 -High-income countries,,2011,467.71356 -High-income countries,,2012,469.939 -High-income countries,,2013,465.38028 -High-income countries,,2014,458.62735 -High-income countries,,2015,447.3899 -High-income countries,,2016,435.8242 -High-income countries,,2017,430.10934 -High-income countries,,2018,420.76245 -High-income countries,,2019,401.36984 -High-income countries,,2020,381.80594 -High-income countries,,2021,387.2416 -High-income countries,,2022,385.2845 -Honduras,HND,2000,263.73627 +Haiti,HTI,2020,568.62744 +Haiti,HTI,2021,572.81555 +Haiti,HTI,2022,534.6535 +High-income countries,,2000,497.98947 +High-income countries,,2001,497.03085 +High-income countries,,2002,497.4434 +High-income countries,,2003,507.51978 +High-income countries,,2004,498.94467 +High-income countries,,2005,499.22586 +High-income countries,,2006,496.14996 +High-income countries,,2007,501.36987 +High-income countries,,2008,495.17072 +High-income countries,,2009,479.0897 +High-income countries,,2010,479.5317 +High-income countries,,2011,478.7199 +High-income countries,,2012,480.07574 +High-income countries,,2013,474.80606 +High-income countries,,2014,468.51764 +High-income countries,,2015,457.3982 +High-income countries,,2016,446.0089 +High-income countries,,2017,440.9906 +High-income countries,,2018,432.45615 +High-income countries,,2019,414.2442 +High-income countries,,2020,393.32025 +High-income countries,,2021,399.64816 +High-income countries,,2022,398.7797 +High-income countries,,2023,378.95743 +Honduras,HND,2000,263.73624 Honduras,HND,2001,345.26855 Honduras,HND,2002,404.81924 -Honduras,HND,2003,405.98288 -Honduras,HND,2004,455.42633 -Honduras,HND,2005,452.07953 -Honduras,HND,2006,429.5302 +Honduras,HND,2003,405.9829 +Honduras,HND,2004,455.42636 +Honduras,HND,2005,452.07956 +Honduras,HND,2006,429.53018 Honduras,HND,2007,426.07315 -Honduras,HND,2008,421.29633 +Honduras,HND,2008,421.2963 Honduras,HND,2009,371.7201 -Honduras,HND,2010,361.11108 -Honduras,HND,2011,382.8648 +Honduras,HND,2010,361.1111 +Honduras,HND,2011,382.86478 Honduras,HND,2012,380.10205 Honduras,HND,2013,393.42267 Honduras,HND,2014,410.84338 -Honduras,HND,2015,386.33936 -Honduras,HND,2016,364.10785 -Honduras,HND,2017,307.06244 +Honduras,HND,2015,386.3394 +Honduras,HND,2016,364.10788 +Honduras,HND,2017,307.0624 Honduras,HND,2018,273.08838 -Honduras,HND,2019,335.13513 -Honduras,HND,2020,318.90875 +Honduras,HND,2019,337.21973 +Honduras,HND,2020,321.16104 Honduras,HND,2021,282.11804 -Honduras,HND,2022,282.26477 +Honduras,HND,2022,293.1181 +Honduras,HND,2023,289.49615 Hong Kong,HKG,2000,792.3865 Hong Kong,HKG,2001,796.3535 Hong Kong,HKG,2002,807.18665 -Hong Kong,HKG,2003,866.4974 -Hong Kong,HKG,2004,829.41974 +Hong Kong,HKG,2003,866.4973 +Hong Kong,HKG,2004,829.4197 Hong Kong,HKG,2005,835.0703 Hong Kong,HKG,2006,828.7493 -Hong Kong,HKG,2007,847.2221 +Hong Kong,HKG,2007,847.22217 Hong Kong,HKG,2008,825.46796 -Hong Kong,HKG,2009,837.13104 +Hong Kong,HKG,2009,837.1311 Hong Kong,HKG,2010,799.8954 -Hong Kong,HKG,2011,838.33716 +Hong Kong,HKG,2011,838.3371 Hong Kong,HKG,2012,812.09454 -Hong Kong,HKG,2013,854.3964 -Hong Kong,HKG,2014,860.5938 +Hong Kong,HKG,2013,854.39636 +Hong Kong,HKG,2014,860.5939 Hong Kong,HKG,2015,814.8442 -Hong Kong,HKG,2016,815.3201 -Hong Kong,HKG,2017,810.5036 -Hong Kong,HKG,2018,790.2652 +Hong Kong,HKG,2016,815.32007 +Hong Kong,HKG,2017,810.50354 +Hong Kong,HKG,2018,790.26526 Hong Kong,HKG,2019,776.6649 -Hong Kong,HKG,2020,691.78467 -Hong Kong,HKG,2021,699.3552 -Hong Kong,HKG,2022,699.49915 -Hungary,HUN,1990,414.55695 -Hungary,HUN,1991,416.8892 -Hungary,HUN,1992,442.86615 -Hungary,HUN,1993,453.96536 -Hungary,HUN,1994,445.40573 -Hungary,HUN,1995,450.02942 -Hungary,HUN,1996,451.69562 -Hungary,HUN,1997,459.03955 -Hungary,HUN,1998,462.6344 -Hungary,HUN,1999,466.29657 +Hong Kong,HKG,2020,684.134 +Hong Kong,HKG,2021,691.6068 +Hong Kong,HKG,2022,682.2715 +Hong Kong,HKG,2023,681.98724 Hungary,HUN,2000,450.9804 Hungary,HUN,2001,439.0445 Hungary,HUN,2002,432.36514 -Hungary,HUN,2003,468.79575 +Hungary,HUN,2003,468.79578 Hungary,HUN,2004,431.32602 -Hungary,HUN,2005,382.65738 +Hungary,HUN,2005,382.65735 Hungary,HUN,2006,389.95816 Hungary,HUN,2007,385.8859 Hungary,HUN,2008,376.06198 Hungary,HUN,2009,343.91534 Hungary,HUN,2010,341.0064 Hungary,HUN,2011,341.19934 -Hungary,HUN,2012,332.75568 +Hungary,HUN,2012,332.75565 Hungary,HUN,2013,317.47604 -Hungary,HUN,2014,300.10205 +Hungary,HUN,2014,300.10208 Hungary,HUN,2015,299.04385 Hungary,HUN,2016,300.1882 Hungary,HUN,2017,290.70474 Hungary,HUN,2018,284.5554 Hungary,HUN,2019,266.04434 -Hungary,HUN,2020,257.44562 -Hungary,HUN,2021,238.50417 -Hungary,HUN,2022,228.74722 -Hungary,HUN,2023,204.18994 -Iceland,ISL,2000,26.281206 +Hungary,HUN,2020,257.4456 +Hungary,HUN,2021,238.50415 +Hungary,HUN,2022,228.89882 +Hungary,HUN,2023,196.22746 +Hungary,HUN,2024,182.81822 +Iceland,ISL,2000,26.281208 Iceland,ISL,2001,25.125628 Iceland,ISL,2002,26.347303 Iceland,ISL,2003,27.251186 @@ -2491,74 +2478,77 @@ Iceland,ISL,2006,27.411167 Iceland,ISL,2007,27.754416 Iceland,ISL,2008,26.927784 Iceland,ISL,2009,27.528429 -Iceland,ISL,2010,27.479092 +Iceland,ISL,2010,27.479094 Iceland,ISL,2011,27.299704 -Iceland,ISL,2012,27.761711 -Iceland,ISL,2013,27.793217 -Iceland,ISL,2014,28.089888 +Iceland,ISL,2012,27.76171 +Iceland,ISL,2013,27.793219 +Iceland,ISL,2014,28.08989 Iceland,ISL,2015,27.493261 Iceland,ISL,2016,27.517887 Iceland,ISL,2017,27.26796 Iceland,ISL,2018,28.440832 Iceland,ISL,2019,28.40909 -Iceland,ISL,2020,28.05717 +Iceland,ISL,2020,28.057173 Iceland,ISL,2021,27.84941 Iceland,ISL,2022,27.679918 +Iceland,ISL,2023,28.330019 India,IND,2000,740.0374 -India,IND,2001,747.3937 -India,IND,2002,755.7575 +India,IND,2001,747.39374 +India,IND,2002,755.75745 India,IND,2003,757.99603 India,IND,2004,723.79425 India,IND,2005,734.0317 -India,IND,2006,723.023 -India,IND,2007,717.8684 +India,IND,2006,723.02295 +India,IND,2007,717.8685 India,IND,2008,733.07904 -India,IND,2009,743.89276 +India,IND,2009,743.8928 India,IND,2010,738.8183 India,IND,2011,721.29596 -India,IND,2012,741.2075 +India,IND,2012,741.2076 India,IND,2013,736.66394 India,IND,2014,741.5031 India,IND,2015,745.78326 -India,IND,2016,749.1475 +India,IND,2016,749.14746 India,IND,2017,739.80005 India,IND,2018,735.122 -India,IND,2019,714.87573 +India,IND,2019,714.8758 India,IND,2020,702.42865 -India,IND,2021,711.69727 -India,IND,2022,705.1314 -India,IND,2023,713.4407 +India,IND,2021,709.8808 +India,IND,2022,702.8098 +India,IND,2023,713.0127 +India,IND,2024,708.3216 Indonesia,IDN,2000,630.66095 Indonesia,IDN,2001,626.03613 -Indonesia,IDN,2002,643.48895 +Indonesia,IDN,2002,643.489 Indonesia,IDN,2003,656.52954 -Indonesia,IDN,2004,653.7106 +Indonesia,IDN,2004,653.7105 Indonesia,IDN,2005,650.538 Indonesia,IDN,2006,668.1159 Indonesia,IDN,2007,663.1402 -Indonesia,IDN,2008,652.9828 +Indonesia,IDN,2008,652.9827 Indonesia,IDN,2009,660.5882 Indonesia,IDN,2010,636.85516 Indonesia,IDN,2011,667.39624 -Indonesia,IDN,2012,687.49994 +Indonesia,IDN,2012,687.5 Indonesia,IDN,2013,682.29865 Indonesia,IDN,2014,691.5888 Indonesia,IDN,2015,696.2759 Indonesia,IDN,2016,689.0313 -Indonesia,IDN,2017,688.49927 +Indonesia,IDN,2017,688.4475 Indonesia,IDN,2018,684.06213 Indonesia,IDN,2019,693.8126 Indonesia,IDN,2020,685.0456 -Indonesia,IDN,2021,684.219 +Indonesia,IDN,2021,684.21906 Indonesia,IDN,2022,675.9309 -Iran,IRN,2000,667.44867 +Indonesia,IDN,2023,682.4301 +Iran,IRN,2000,667.4487 Iran,IRN,2001,667.7639 -Iran,IRN,2002,651.81226 +Iran,IRN,2002,651.8122 Iran,IRN,2003,648.9944 Iran,IRN,2004,641.5581 -Iran,IRN,2005,636.30194 +Iran,IRN,2005,636.302 Iran,IRN,2006,624.7221 -Iran,IRN,2007,630.8586 +Iran,IRN,2007,630.85864 Iran,IRN,2008,662.35925 Iran,IRN,2009,666.832 Iran,IRN,2010,661.2356 @@ -2567,82 +2557,75 @@ Iran,IRN,2012,648.34766 Iran,IRN,2013,633.6244 Iran,IRN,2014,644.16064 Iran,IRN,2015,650.835 -Iran,IRN,2016,642.13324 +Iran,IRN,2016,642.1332 Iran,IRN,2017,641.6751 -Iran,IRN,2018,657.9157 -Iran,IRN,2019,607.73303 -Iran,IRN,2020,632.76984 -Iran,IRN,2021,656.5604 -Iran,IRN,2022,665.15375 -Iran,IRN,2023,655.12494 +Iran,IRN,2018,657.8838 +Iran,IRN,2019,607.8149 +Iran,IRN,2020,632.8104 +Iran,IRN,2021,659.9703 +Iran,IRN,2022,655.17053 +Iran,IRN,2023,641.7322 +Iran,IRN,2024,641.93616 Iraq,IRQ,2000,658.3072 -Iraq,IRQ,2001,656.43414 +Iraq,IRQ,2001,656.4341 Iraq,IRQ,2002,657.11755 Iraq,IRQ,2003,655.4889 -Iraq,IRQ,2004,652.21423 -Iraq,IRQ,2005,554.9342 -Iraq,IRQ,2006,571.68195 +Iraq,IRQ,2004,652.2143 +Iraq,IRQ,2005,554.93427 +Iraq,IRQ,2006,571.6819 Iraq,IRQ,2007,574.6089 -Iraq,IRQ,2008,631.2364 +Iraq,IRQ,2008,631.23645 Iraq,IRQ,2009,635.93134 -Iraq,IRQ,2010,632.7949 -Iraq,IRQ,2011,638.7255 -Iraq,IRQ,2012,630.51514 -Iraq,IRQ,2013,638.6799 -Iraq,IRQ,2014,666.5192 -Iraq,IRQ,2015,667.7818 -Iraq,IRQ,2016,669.4555 -Iraq,IRQ,2017,681.0798 -Iraq,IRQ,2018,682.68646 -Iraq,IRQ,2019,660.03406 -Iraq,IRQ,2020,668.2664 -Iraq,IRQ,2021,671.6955 -Iraq,IRQ,2022,688.81396 -Ireland,IRL,1990,695.285 -Ireland,IRL,1991,692.8236 -Ireland,IRL,1992,704.7619 -Ireland,IRL,1993,694.5996 -Ireland,IRL,1994,687.9094 -Ireland,IRL,1995,692.0888 -Ireland,IRL,1996,676.20557 -Ireland,IRL,1997,668.70233 -Ireland,IRL,1998,656.5947 -Ireland,IRL,1999,640.42365 -Ireland,IRL,2000,632.9381 +Iraq,IRQ,2010,616.6428 +Iraq,IRQ,2011,640.2597 +Iraq,IRQ,2012,612.34247 +Iraq,IRQ,2013,620.2037 +Iraq,IRQ,2014,645.22675 +Iraq,IRQ,2015,654.44507 +Iraq,IRQ,2016,648.4804 +Iraq,IRQ,2017,666.80914 +Iraq,IRQ,2018,682.0445 +Iraq,IRQ,2019,668.86554 +Iraq,IRQ,2020,674.1803 +Iraq,IRQ,2021,677.73535 +Iraq,IRQ,2022,688.70465 +Iraq,IRQ,2023,689.4005 +Ireland,IRL,2000,632.9382 Ireland,IRL,2001,641.57855 Ireland,IRL,2002,615.7258 Ireland,IRL,2003,595.89044 Ireland,IRL,2004,591.73944 Ireland,IRL,2005,593.4324 Ireland,IRL,2006,560.02954 -Ireland,IRL,2007,536.18066 -Ireland,IRL,2008,517.56445 +Ireland,IRL,2007,536.1807 +Ireland,IRL,2008,517.5644 Ireland,IRL,2009,487.47314 Ireland,IRL,2010,479.1889 Ireland,IRL,2011,457.0586 Ireland,IRL,2012,484.31155 -Ireland,IRL,2013,464.36957 +Ireland,IRL,2013,464.3696 Ireland,IRL,2014,447.63382 -Ireland,IRL,2015,441.5121 +Ireland,IRL,2015,441.51212 Ireland,IRL,2016,444.37085 Ireland,IRL,2017,405.74036 -Ireland,IRL,2018,364.0777 +Ireland,IRL,2018,364.07767 Ireland,IRL,2019,313.25302 -Ireland,IRL,2020,292.8661 -Ireland,IRL,2021,353.81693 -Ireland,IRL,2022,333.23404 -Ireland,IRL,2023,290.805 +Ireland,IRL,2020,292.77448 +Ireland,IRL,2021,353.7049 +Ireland,IRL,2022,332.9366 +Ireland,IRL,2023,283.801 +Ireland,IRL,2024,279.70218 Israel,ISR,2000,794.421 Israel,ISR,2001,807.50854 Israel,ISR,2002,811.6484 -Israel,ISR,2003,810.79926 +Israel,ISR,2003,810.7993 Israel,ISR,2004,803.72253 -Israel,ISR,2005,796.13245 -Israel,ISR,2006,783.42566 +Israel,ISR,2005,796.1325 +Israel,ISR,2006,783.4256 Israel,ISR,2007,779.32697 Israel,ISR,2008,758.40814 Israel,ISR,2009,753.8827 -Israel,ISR,2010,740.7344 +Israel,ISR,2010,740.73444 Israel,ISR,2011,744.4631 Israel,ISR,2012,761.7839 Israel,ISR,2013,719.8304 @@ -2650,139 +2633,133 @@ Israel,ISR,2014,705.3894 Israel,ISR,2015,692.667 Israel,ISR,2016,662.13995 Israel,ISR,2017,653.17316 -Israel,ISR,2018,644.2601 -Israel,ISR,2019,635.5063 -Israel,ISR,2020,615.2886 -Israel,ISR,2021,598.1524 -Israel,ISR,2022,582.9271 -Italy,ITA,1990,564.25055 -Italy,ITA,1991,533.7637 -Italy,ITA,1992,527.214 -Italy,ITA,1993,517.5662 -Italy,ITA,1994,517.8142 -Italy,ITA,1995,541.64386 -Italy,ITA,1996,525.377 -Italy,ITA,1997,519.8377 -Italy,ITA,1998,519.7508 -Italy,ITA,1999,500.86792 -Italy,ITA,2000,501.66705 -Italy,ITA,2001,498.67596 -Italy,ITA,2002,523.0786 +Israel,ISR,2018,643.5816 +Israel,ISR,2019,638.96606 +Israel,ISR,2020,619.81055 +Israel,ISR,2021,603.69104 +Israel,ISR,2022,595.85913 +Israel,ISR,2023,567.26245 +Italy,ITA,2000,501.66702 +Italy,ITA,2001,498.676 +Italy,ITA,2002,523.07855 Italy,ITA,2003,523.24725 -Italy,ITA,2004,510.9802 +Italy,ITA,2004,510.98022 Italy,ITA,2005,510.16528 -Italy,ITA,2006,506.75696 -Italy,ITA,2007,504.7522 +Italy,ITA,2006,506.757 +Italy,ITA,2007,504.75223 Italy,ITA,2008,485.50888 -Italy,ITA,2009,458.58368 -Italy,ITA,2010,446.41473 +Italy,ITA,2009,458.58365 +Italy,ITA,2010,446.4147 Italy,ITA,2011,445.52063 -Italy,ITA,2012,438.1033 -Italy,ITA,2013,398.85327 -Italy,ITA,2014,379.2355 +Italy,ITA,2012,438.10333 +Italy,ITA,2013,398.85324 +Italy,ITA,2014,379.23553 Italy,ITA,2015,398.60562 -Italy,ITA,2016,389.5086 +Italy,ITA,2016,389.50864 Italy,ITA,2017,392.75568 -Italy,ITA,2018,366.8112 -Italy,ITA,2019,351.28802 -Italy,ITA,2020,334.27258 -Italy,ITA,2021,339.27637 -Italy,ITA,2022,378.44693 -Italy,ITA,2023,330.71823 +Italy,ITA,2018,366.81116 +Italy,ITA,2019,351.288 +Italy,ITA,2020,334.2726 +Italy,ITA,2021,339.27634 +Italy,ITA,2022,378.16324 +Italy,ITA,2023,323.4485 +Italy,ITA,2024,287.5335 Jamaica,JAM,2000,633.678 Jamaica,JAM,2001,642.1208 Jamaica,JAM,2002,645.1149 Jamaica,JAM,2003,643.45404 -Jamaica,JAM,2004,639.99994 +Jamaica,JAM,2004,640 Jamaica,JAM,2005,637.2156 Jamaica,JAM,2006,636 Jamaica,JAM,2007,628.7129 -Jamaica,JAM,2008,619.38525 -Jamaica,JAM,2009,615.9091 -Jamaica,JAM,2010,615.7408 +Jamaica,JAM,2008,619.3853 +Jamaica,JAM,2009,615.90906 +Jamaica,JAM,2010,615.7407 Jamaica,JAM,2011,607.79816 Jamaica,JAM,2012,600.92804 Jamaica,JAM,2013,603.7296 Jamaica,JAM,2014,600.9389 -Jamaica,JAM,2015,600.4673 +Jamaica,JAM,2015,600.4672 Jamaica,JAM,2016,587.9733 Jamaica,JAM,2017,579.9086 Jamaica,JAM,2018,558.0808 Jamaica,JAM,2019,571.1207 Jamaica,JAM,2020,553.7383 Jamaica,JAM,2021,556.32184 -Jamaica,JAM,2022,555.55554 +Jamaica,JAM,2022,561.24725 Japan,JPN,2000,424.4273 Japan,JPN,2001,424.01718 Japan,JPN,2002,433.7259 -Japan,JPN,2003,476.31772 -Japan,JPN,2004,450.2627 -Japan,JPN,2005,461.80194 -Japan,JPN,2006,447.24527 -Japan,JPN,2007,470.19345 +Japan,JPN,2003,476.31775 +Japan,JPN,2004,450.26266 +Japan,JPN,2005,461.80197 +Japan,JPN,2006,447.24524 +Japan,JPN,2007,470.19348 Japan,JPN,2008,485.2243 Japan,JPN,2009,460.20715 -Japan,JPN,2010,446.6627 +Japan,JPN,2010,446.66272 Japan,JPN,2011,505.34326 Japan,JPN,2012,594.527 Japan,JPN,2013,595.9035 Japan,JPN,2014,591.5293 Japan,JPN,2015,578.0492 -Japan,JPN,2016,569.65515 -Japan,JPN,2017,553.7536 -Japan,JPN,2018,535.99567 -Japan,JPN,2019,523.98175 -Japan,JPN,2020,525.71814 -Japan,JPN,2021,508.52692 -Japan,JPN,2022,512.80536 -Japan,JPN,2023,485.39236 -Jordan,JOR,2000,658.0733 +Japan,JPN,2016,567.03955 +Japan,JPN,2017,558.02985 +Japan,JPN,2018,541.4662 +Japan,JPN,2019,528.48956 +Japan,JPN,2020,527.7489 +Japan,JPN,2021,514.49255 +Japan,JPN,2022,519.6191 +Japan,JPN,2023,493.58505 +Japan,JPN,2024,482.32416 +Jordan,JOR,2000,658.07324 Jordan,JOR,2001,657.82495 Jordan,JOR,2002,656.40393 Jordan,JOR,2003,657.5343 Jordan,JOR,2004,670.759 -Jordan,JOR,2005,672.5388 +Jordan,JOR,2005,672.5389 Jordan,JOR,2006,678.95685 -Jordan,JOR,2007,682.7747 +Jordan,JOR,2007,682.7748 Jordan,JOR,2008,684.099 Jordan,JOR,2009,686.27454 Jordan,JOR,2010,679.97296 -Jordan,JOR,2011,664.6175 +Jordan,JOR,2011,664.61743 Jordan,JOR,2012,661.2417 -Jordan,JOR,2013,664.54236 -Jordan,JOR,2014,657.9091 -Jordan,JOR,2015,666.8416 +Jordan,JOR,2013,664.5423 +Jordan,JOR,2014,657.9092 +Jordan,JOR,2015,666.84155 Jordan,JOR,2016,659.8881 -Jordan,JOR,2017,643.40704 -Jordan,JOR,2018,621.0113 +Jordan,JOR,2017,643.4071 +Jordan,JOR,2018,621.0112 Jordan,JOR,2019,596.68774 Jordan,JOR,2020,566.6666 Jordan,JOR,2021,540.26215 -Jordan,JOR,2022,540.92365 -Kazakhstan,KAZ,2000,781.1773 +Jordan,JOR,2022,539.21136 +Kazakhstan,KAZ,2000,781.17737 Kazakhstan,KAZ,2001,783.49585 -Kazakhstan,KAZ,2002,775.24426 +Kazakhstan,KAZ,2002,775.2443 Kazakhstan,KAZ,2003,793.7676 -Kazakhstan,KAZ,2004,812.84546 +Kazakhstan,KAZ,2004,812.8454 Kazakhstan,KAZ,2005,811.54297 Kazakhstan,KAZ,2006,804.63293 -Kazakhstan,KAZ,2007,802.5319 +Kazakhstan,KAZ,2007,802.532 Kazakhstan,KAZ,2008,832.10956 -Kazakhstan,KAZ,2009,848.59644 -Kazakhstan,KAZ,2010,835.4507 +Kazakhstan,KAZ,2009,848.5964 +Kazakhstan,KAZ,2010,835.45074 Kazakhstan,KAZ,2011,841.41833 Kazakhstan,KAZ,2012,852.3504 Kazakhstan,KAZ,2013,854.55133 -Kazakhstan,KAZ,2014,852.07104 -Kazakhstan,KAZ,2015,837.9706 -Kazakhstan,KAZ,2016,816.7601 -Kazakhstan,KAZ,2017,832.81604 -Kazakhstan,KAZ,2018,843.9452 -Kazakhstan,KAZ,2019,842.0361 -Kazakhstan,KAZ,2020,838.0742 -Kazakhstan,KAZ,2021,839.58984 -Kazakhstan,KAZ,2022,830.40875 -Kazakhstan,KAZ,2023,821.3909 +Kazakhstan,KAZ,2014,852.4781 +Kazakhstan,KAZ,2015,837.0799 +Kazakhstan,KAZ,2016,815.5114 +Kazakhstan,KAZ,2017,833.6889 +Kazakhstan,KAZ,2018,842.91974 +Kazakhstan,KAZ,2019,841.26984 +Kazakhstan,KAZ,2020,838.1663 +Kazakhstan,KAZ,2021,833.8547 +Kazakhstan,KAZ,2022,831.5557 +Kazakhstan,KAZ,2023,821.8996 +Kazakhstan,KAZ,2024,801.9483 Kenya,KEN,2000,403.7123 Kenya,KEN,2001,282.7443 Kenya,KEN,2002,188.34952 @@ -2791,22 +2768,23 @@ Kenya,KEN,2004,228.8961 Kenya,KEN,2005,250.74626 Kenya,KEN,2006,274.3009 Kenya,KEN,2007,203.89806 -Kenya,KEN,2008,214.63414 +Kenya,KEN,2008,214.63416 Kenya,KEN,2009,290.07632 Kenya,KEN,2010,199.72067 Kenya,KEN,2011,233.71648 Kenya,KEN,2012,179.92656 Kenya,KEN,2013,174.50494 Kenya,KEN,2014,188.04349 -Kenya,KEN,2015,103.97552 +Kenya,KEN,2015,103.97553 Kenya,KEN,2016,108.47107 -Kenya,KEN,2017,163.43489 -Kenya,KEN,2018,104.702736 -Kenya,KEN,2019,95.52496 -Kenya,KEN,2020,68.103455 -Kenya,KEN,2021,82.021545 -Kenya,KEN,2022,83.333336 -Kenya,KEN,2023,70.491806 +Kenya,KEN,2017,163.4349 +Kenya,KEN,2018,117.69912 +Kenya,KEN,2019,102.94118 +Kenya,KEN,2020,71.9178 +Kenya,KEN,2021,96.12278 +Kenya,KEN,2022,109.567894 +Kenya,KEN,2023,96.950745 +Kenya,KEN,2024,84.824905 Kiribati,KIR,2000,1000 Kiribati,KIR,2001,500 Kiribati,KIR,2002,500 @@ -2826,34 +2804,35 @@ Kiribati,KIR,2015,666.6667 Kiribati,KIR,2016,666.6667 Kiribati,KIR,2017,666.6667 Kiribati,KIR,2018,666.6667 -Kiribati,KIR,2019,666.6667 -Kiribati,KIR,2020,666.6667 -Kiribati,KIR,2021,666.6667 -Kiribati,KIR,2022,666.6667 +Kiribati,KIR,2019,500 +Kiribati,KIR,2020,500 +Kiribati,KIR,2021,500 +Kiribati,KIR,2022,500 Kosovo,OWID_KOS,2000,989.86487 Kosovo,OWID_KOS,2001,994.63806 -Kosovo,OWID_KOS,2002,994.60913 +Kosovo,OWID_KOS,2002,994.6092 Kosovo,OWID_KOS,2003,994.4903 Kosovo,OWID_KOS,2004,980.39215 -Kosovo,OWID_KOS,2005,984.3048 -Kosovo,OWID_KOS,2006,986.45605 +Kosovo,OWID_KOS,2005,984.3049 +Kosovo,OWID_KOS,2006,986.456 Kosovo,OWID_KOS,2007,989.6481 Kosovo,OWID_KOS,2008,994.1861 -Kosovo,OWID_KOS,2009,985.91547 -Kosovo,OWID_KOS,2010,976.78925 -Kosovo,OWID_KOS,2011,991.37933 -Kosovo,OWID_KOS,2012,993.2659 -Kosovo,OWID_KOS,2013,989.26373 -Kosovo,OWID_KOS,2014,983.42554 -Kosovo,OWID_KOS,2015,988.5434 -Kosovo,OWID_KOS,2016,969.8996 +Kosovo,OWID_KOS,2009,985.9155 +Kosovo,OWID_KOS,2010,976.7892 +Kosovo,OWID_KOS,2011,991.3793 +Kosovo,OWID_KOS,2012,993.266 +Kosovo,OWID_KOS,2013,989.2638 +Kosovo,OWID_KOS,2014,983.4255 +Kosovo,OWID_KOS,2015,988.54333 +Kosovo,OWID_KOS,2016,969.8997 Kosovo,OWID_KOS,2017,979.72974 -Kosovo,OWID_KOS,2018,961.08295 -Kosovo,OWID_KOS,2019,960.6298 +Kosovo,OWID_KOS,2018,961.0829 +Kosovo,OWID_KOS,2019,960.62994 Kosovo,OWID_KOS,2020,958.209 -Kosovo,OWID_KOS,2021,945.0073 -Kosovo,OWID_KOS,2022,945.0073 -Kosovo,OWID_KOS,2023,894.6541 +Kosovo,OWID_KOS,2021,945.00726 +Kosovo,OWID_KOS,2022,945.00726 +Kosovo,OWID_KOS,2023,918.16693 +Kosovo,OWID_KOS,2024,958.7156 Kuwait,KWT,2000,653.8818 Kuwait,KWT,2001,653.9359 Kuwait,KWT,2002,655.2103 @@ -2861,8 +2840,8 @@ Kuwait,KWT,2003,654.77386 Kuwait,KWT,2004,654.38684 Kuwait,KWT,2005,654.6993 Kuwait,KWT,2006,653.78156 -Kuwait,KWT,2007,653.74365 -Kuwait,KWT,2008,653.14014 +Kuwait,KWT,2007,653.7436 +Kuwait,KWT,2008,653.1401 Kuwait,KWT,2009,654.2653 Kuwait,KWT,2010,653.34033 Kuwait,KWT,2011,652.6279 @@ -2871,36 +2850,39 @@ Kuwait,KWT,2013,651.9101 Kuwait,KWT,2014,650.9593 Kuwait,KWT,2015,650.5124 Kuwait,KWT,2016,650.07135 -Kuwait,KWT,2017,650.2815 -Kuwait,KWT,2018,649.54144 -Kuwait,KWT,2019,648.59467 -Kuwait,KWT,2020,648.5619 -Kuwait,KWT,2021,649.17053 -Kuwait,KWT,2022,649.1959 -Kuwait,KWT,2023,649.1634 +Kuwait,KWT,2017,650.6024 +Kuwait,KWT,2018,650.0742 +Kuwait,KWT,2019,649.52026 +Kuwait,KWT,2020,650.18756 +Kuwait,KWT,2021,649.6287 +Kuwait,KWT,2022,636.8859 +Kuwait,KWT,2023,636.90546 +Kuwait,KWT,2024,637.23474 Kyrgyzstan,KGZ,2000,101.577286 Kyrgyzstan,KGZ,2001,110.735985 -Kyrgyzstan,KGZ,2002,107.08782 -Kyrgyzstan,KGZ,2003,89.378235 -Kyrgyzstan,KGZ,2004,89.72772 +Kyrgyzstan,KGZ,2002,107.08783 +Kyrgyzstan,KGZ,2003,89.37824 +Kyrgyzstan,KGZ,2004,89.72773 Kyrgyzstan,KGZ,2005,132.34302 Kyrgyzstan,KGZ,2006,135.39519 -Kyrgyzstan,KGZ,2007,138.09523 -Kyrgyzstan,KGZ,2008,142.00172 +Kyrgyzstan,KGZ,2007,138.09525 +Kyrgyzstan,KGZ,2008,142.00171 Kyrgyzstan,KGZ,2009,81.96722 Kyrgyzstan,KGZ,2010,85.90492 -Kyrgyzstan,KGZ,2011,75.28315 +Kyrgyzstan,KGZ,2011,75.28314 Kyrgyzstan,KGZ,2012,78.50964 Kyrgyzstan,KGZ,2013,82.61494 -Kyrgyzstan,KGZ,2014,98.4061 -Kyrgyzstan,KGZ,2015,154.79875 +Kyrgyzstan,KGZ,2014,98.40609 +Kyrgyzstan,KGZ,2015,154.79877 Kyrgyzstan,KGZ,2016,139.1635 Kyrgyzstan,KGZ,2017,97.59271 -Kyrgyzstan,KGZ,2018,102.69576 -Kyrgyzstan,KGZ,2019,98.19639 +Kyrgyzstan,KGZ,2018,95.26896 +Kyrgyzstan,KGZ,2019,98.196396 Kyrgyzstan,KGZ,2020,105.19481 Kyrgyzstan,KGZ,2021,151.25494 -Kyrgyzstan,KGZ,2022,147.2924 +Kyrgyzstan,KGZ,2022,146.88857 +Kyrgyzstan,KGZ,2023,150.76476 +Kyrgyzstan,KGZ,2024,163.70107 Laos,LAO,2000,87.67123 Laos,LAO,2001,87.19346 Laos,LAO,2002,89.63586 @@ -2912,101 +2894,95 @@ Laos,LAO,2007,23.738873 Laos,LAO,2008,24.19355 Laos,LAO,2009,23.738873 Laos,LAO,2010,23.923447 -Laos,LAO,2011,24.143301 +Laos,LAO,2011,24.143303 Laos,LAO,2012,23.97525 Laos,LAO,2013,23.437502 -Laos,LAO,2014,23.809526 +Laos,LAO,2014,23.809525 Laos,LAO,2015,148.51486 Laos,LAO,2016,302.86386 Laos,LAO,2017,338.5903 Laos,LAO,2018,336.14178 -Laos,LAO,2019,350.51218 +Laos,LAO,2019,350.51215 Laos,LAO,2020,278.028 -Laos,LAO,2021,260.46304 -Laos,LAO,2022,265.50595 -Latin America and Caribbean (Ember),,2000,271.04404 -Latin America and Caribbean (Ember),,2001,286.78122 -Latin America and Caribbean (Ember),,2002,284.4708 -Latin America and Caribbean (Ember),,2003,287.30048 -Latin America and Caribbean (Ember),,2004,289.01874 -Latin America and Caribbean (Ember),,2005,286.85715 -Latin America and Caribbean (Ember),,2006,283.74075 -Latin America and Caribbean (Ember),,2007,289.75433 -Latin America and Caribbean (Ember),,2008,289.60675 -Latin America and Caribbean (Ember),,2009,288.79007 -Latin America and Caribbean (Ember),,2010,297.49588 -Latin America and Caribbean (Ember),,2011,290.5889 -Latin America and Caribbean (Ember),,2012,309.74835 -Latin America and Caribbean (Ember),,2013,325.4778 -Latin America and Caribbean (Ember),,2014,332.70755 -Latin America and Caribbean (Ember),,2015,333.67105 -Latin America and Caribbean (Ember),,2016,324.0271 -Latin America and Caribbean (Ember),,2017,309.50476 -Latin America and Caribbean (Ember),,2018,301.1136 -Latin America and Caribbean (Ember),,2019,302.97702 -Latin America and Caribbean (Ember),,2020,280.47595 -Latin America and Caribbean (Ember),,2021,286.60123 -Latin America and Caribbean (Ember),,2022,260.2797 -Latin America and Caribbean (Ember),,2023,258.5933 -Latvia,LVA,1990,153.38345 -Latvia,LVA,1991,196.46019 -Latvia,LVA,1992,153.64583 -Latvia,LVA,1993,150.12721 -Latvia,LVA,1994,164.41441 -Latvia,LVA,1995,150.37595 -Latvia,LVA,1996,230.76926 -Latvia,LVA,1997,161.86252 -Latvia,LVA,1998,129.31035 -Latvia,LVA,1999,165.45012 -Latvia,LVA,2000,147.34302 +Laos,LAO,2021,260.35632 +Laos,LAO,2022,237.26488 +Laos,LAO,2023,232.12238 +Latin America and Caribbean (Ember),,2000,271.59796 +Latin America and Caribbean (Ember),,2001,287.23407 +Latin America and Caribbean (Ember),,2002,284.77707 +Latin America and Caribbean (Ember),,2003,287.87753 +Latin America and Caribbean (Ember),,2004,289.35107 +Latin America and Caribbean (Ember),,2005,286.8605 +Latin America and Caribbean (Ember),,2006,284.07373 +Latin America and Caribbean (Ember),,2007,290.33307 +Latin America and Caribbean (Ember),,2008,290.48334 +Latin America and Caribbean (Ember),,2009,289.31586 +Latin America and Caribbean (Ember),,2010,297.98254 +Latin America and Caribbean (Ember),,2011,290.6306 +Latin America and Caribbean (Ember),,2012,310.35095 +Latin America and Caribbean (Ember),,2013,326.09277 +Latin America and Caribbean (Ember),,2014,332.25372 +Latin America and Caribbean (Ember),,2015,334.16672 +Latin America and Caribbean (Ember),,2016,323.59064 +Latin America and Caribbean (Ember),,2017,310.94345 +Latin America and Caribbean (Ember),,2018,302.5968 +Latin America and Caribbean (Ember),,2019,303.99277 +Latin America and Caribbean (Ember),,2020,281.83353 +Latin America and Caribbean (Ember),,2021,285.80325 +Latin America and Caribbean (Ember),,2022,259.51196 +Latin America and Caribbean (Ember),,2023,258.13376 +Latin America and Caribbean (Ember),,2024,255.2324 +Latvia,LVA,2000,147.343 Latvia,LVA,2001,147.54099 Latvia,LVA,2002,166.24686 Latvia,LVA,2003,171.28464 Latvia,LVA,2004,136.46056 Latvia,LVA,2005,130.0813 -Latvia,LVA,2006,171.77913 +Latvia,LVA,2006,171.77914 Latvia,LVA,2007,163.86554 Latvia,LVA,2008,157.49525 Latvia,LVA,2009,147.21722 Latvia,LVA,2010,179.48718 Latvia,LVA,2011,197.04434 Latvia,LVA,2012,145.8671 -Latvia,LVA,2013,187.09676 +Latvia,LVA,2013,187.09677 Latvia,LVA,2014,202.33463 -Latvia,LVA,2015,220.21663 -Latvia,LVA,2016,204.04982 -Latvia,LVA,2017,140.77023 -Latvia,LVA,2018,215.7738 -Latvia,LVA,2019,223.60251 +Latvia,LVA,2015,220.21661 +Latvia,LVA,2016,204.04984 +Latvia,LVA,2017,140.77025 +Latvia,LVA,2018,215.77382 +Latvia,LVA,2019,223.6025 Latvia,LVA,2020,176.57343 Latvia,LVA,2021,176.06837 -Latvia,LVA,2022,136.27257 -Latvia,LVA,2023,123.2 +Latvia,LVA,2022,135.18886 +Latvia,LVA,2023,119.309265 +Latvia,LVA,2024,136.22292 Lebanon,LBN,2000,630.15076 Lebanon,LBN,2001,637.186 Lebanon,LBN,2002,621.645 Lebanon,LBN,2003,591.64734 -Lebanon,LBN,2004,597.02795 -Lebanon,LBN,2005,601.023 +Lebanon,LBN,2004,597.028 +Lebanon,LBN,2005,601.0231 Lebanon,LBN,2006,619.594 -Lebanon,LBN,2007,622.5166 +Lebanon,LBN,2007,622.51654 Lebanon,LBN,2008,639.3574 Lebanon,LBN,2009,626.34827 -Lebanon,LBN,2010,619.9873 +Lebanon,LBN,2010,619.98724 Lebanon,LBN,2011,626.9939 Lebanon,LBN,2012,619.07654 -Lebanon,LBN,2013,615.4256 -Lebanon,LBN,2014,650.5286 +Lebanon,LBN,2013,615.42554 +Lebanon,LBN,2014,650.5287 Lebanon,LBN,2015,641.0944 Lebanon,LBN,2016,643.9979 Lebanon,LBN,2017,643.7647 -Lebanon,LBN,2018,644.40265 -Lebanon,LBN,2019,625.46295 -Lebanon,LBN,2020,615.6798 -Lebanon,LBN,2021,599.61316 -Lebanon,LBN,2022,599.005 +Lebanon,LBN,2018,644.2669 +Lebanon,LBN,2019,625.7527 +Lebanon,LBN,2020,615.8904 +Lebanon,LBN,2021,599.6131 +Lebanon,LBN,2022,379.8627 +Lebanon,LBN,2023,369.46902 Lesotho,LSO,2000,34.482758 -Lesotho,LSO,2001,35.714287 +Lesotho,LSO,2001,35.714283 Lesotho,LSO,2002,32.258064 Lesotho,LSO,2003,31.25 Lesotho,LSO,2004,33.333332 @@ -3014,27 +2990,27 @@ Lesotho,LSO,2005,21.73913 Lesotho,LSO,2006,19.23077 Lesotho,LSO,2007,18.181818 Lesotho,LSO,2008,17.241379 -Lesotho,LSO,2009,30.76923 -Lesotho,LSO,2010,28.985508 +Lesotho,LSO,2009,30.769232 +Lesotho,LSO,2010,28.985506 Lesotho,LSO,2011,20.408163 Lesotho,LSO,2012,20.408163 Lesotho,LSO,2013,19.607843 Lesotho,LSO,2014,19.23077 Lesotho,LSO,2015,18.867926 Lesotho,LSO,2016,19.23077 -Lesotho,LSO,2017,18.518518 -Lesotho,LSO,2018,18.518518 -Lesotho,LSO,2019,20 -Lesotho,LSO,2020,20 -Lesotho,LSO,2021,20 -Lesotho,LSO,2022,20 -Liberia,LBR,2000,650 -Liberia,LBR,2001,650 -Liberia,LBR,2002,650 -Liberia,LBR,2003,650 -Liberia,LBR,2004,650 -Liberia,LBR,2005,650 -Liberia,LBR,2006,650 +Lesotho,LSO,2017,18.518517 +Lesotho,LSO,2018,19.23077 +Lesotho,LSO,2019,25.641026 +Lesotho,LSO,2020,23.255814 +Lesotho,LSO,2021,18.867926 +Lesotho,LSO,2022,20.833334 +Liberia,LBR,2000,649.99994 +Liberia,LBR,2001,649.99994 +Liberia,LBR,2002,649.99994 +Liberia,LBR,2003,649.99994 +Liberia,LBR,2004,649.99994 +Liberia,LBR,2005,649.99994 +Liberia,LBR,2006,649.99994 Liberia,LBR,2007,640 Liberia,LBR,2008,640 Liberia,LBR,2009,640 @@ -3043,16 +3019,17 @@ Liberia,LBR,2011,666.6666 Liberia,LBR,2012,666.6666 Liberia,LBR,2013,666.6666 Liberia,LBR,2014,666.6666 -Liberia,LBR,2015,645.16125 +Liberia,LBR,2015,645.1613 Liberia,LBR,2016,666.6666 Liberia,LBR,2017,657.1429 -Liberia,LBR,2018,642.8572 -Liberia,LBR,2019,655.17236 -Liberia,LBR,2020,258.82352 -Liberia,LBR,2021,217.94873 -Liberia,LBR,2022,227.84811 -Libya,LBY,2000,707.7419 -Libya,LBY,2001,707.635 +Liberia,LBR,2018,439.0244 +Liberia,LBR,2019,452.38095 +Liberia,LBR,2020,466.66666 +Liberia,LBR,2021,421.0526 +Liberia,LBR,2022,435.89746 +Liberia,LBR,2023,435.89746 +Libya,LBY,2000,707.74194 +Libya,LBY,2001,707.63495 Libya,LBY,2002,705.6474 Libya,LBY,2003,704.4855 Libya,LBY,2004,700.99005 @@ -3060,44 +3037,35 @@ Libya,LBY,2005,718.63806 Libya,LBY,2006,747.8788 Libya,LBY,2007,757.91077 Libya,LBY,2008,750.9592 -Libya,LBY,2009,746.8729 -Libya,LBY,2010,764.4349 -Libya,LBY,2011,793.54095 +Libya,LBY,2009,746.873 +Libya,LBY,2010,764.4348 +Libya,LBY,2011,793.5409 Libya,LBY,2012,794.98395 Libya,LBY,2013,790.1397 Libya,LBY,2014,779.75085 Libya,LBY,2015,780.3253 -Libya,LBY,2016,822.7244 +Libya,LBY,2016,822.72437 Libya,LBY,2017,813.4015 Libya,LBY,2018,806.61206 -Libya,LBY,2019,809.99133 +Libya,LBY,2019,809.9913 Libya,LBY,2020,824.06775 -Libya,LBY,2021,824.9516 -Libya,LBY,2022,818.6922 -Lithuania,LTU,1990,190.49297 -Lithuania,LTU,1991,212.80217 -Lithuania,LTU,1992,113.20754 -Lithuania,LTU,1993,60.301502 -Lithuania,LTU,1994,103.483604 -Lithuania,LTU,1995,65.08875 -Lithuania,LTU,1996,73.89163 -Lithuania,LTU,1997,90.3405 -Lithuania,LTU,1998,122.448975 -Lithuania,LTU,1999,124.617744 -Lithuania,LTU,2000,103.32433 +Libya,LBY,2021,831.5608 +Libya,LBY,2022,830.5326 +Libya,LBY,2023,830.5326 +Lithuania,LTU,2000,103.32435 Lithuania,LTU,2001,89.83287 -Lithuania,LTU,2002,76.92308 +Lithuania,LTU,2002,76.92307 Lithuania,LTU,2003,71.7322 Lithuania,LTU,2004,77.333336 -Lithuania,LTU,2005,110.263535 +Lithuania,LTU,2005,110.26353 Lithuania,LTU,2006,110.19056 Lithuania,LTU,2007,104.75483 Lithuania,LTU,2008,103.52588 Lithuania,LTU,2009,104.508194 Lithuania,LTU,2010,368 -Lithuania,LTU,2011,322.35297 +Lithuania,LTU,2011,322.35294 Lithuania,LTU,2012,322.22223 -Lithuania,LTU,2013,293.55606 +Lithuania,LTU,2013,293.5561 Lithuania,LTU,2014,284.55283 Lithuania,LTU,2015,295.50827 Lithuania,LTU,2016,234.15977 @@ -3105,66 +3073,58 @@ Lithuania,LTU,2017,182.58426 Lithuania,LTU,2018,178.08218 Lithuania,LTU,2019,170.65869 Lithuania,LTU,2020,231.91492 -Lithuania,LTU,2021,220.9302 -Lithuania,LTU,2022,177.18446 -Lithuania,LTU,2023,160.07195 -Low-income countries,,2000,398.98727 -Low-income countries,,2001,394.40652 -Low-income countries,,2002,401.39484 +Lithuania,LTU,2021,220.93022 +Lithuania,LTU,2022,180.69307 +Lithuania,LTU,2023,120.9213 +Lithuania,LTU,2024,139.34425 +Low-income countries,,2000,398.9872 +Low-income countries,,2001,394.40646 +Low-income countries,,2002,401.39493 Low-income countries,,2003,414.8961 -Low-income countries,,2004,397.3956 +Low-income countries,,2004,397.39554 Low-income countries,,2005,399.94022 -Low-income countries,,2006,407.35782 +Low-income countries,,2006,407.35785 Low-income countries,,2007,394.8411 -Low-income countries,,2008,412.20288 +Low-income countries,,2008,412.2029 Low-income countries,,2009,428.12762 -Low-income countries,,2010,417.76035 -Low-income countries,,2011,386.21658 -Low-income countries,,2012,360.86804 -Low-income countries,,2013,329.53125 +Low-income countries,,2010,417.7603 +Low-income countries,,2011,386.2166 +Low-income countries,,2012,360.86807 +Low-income countries,,2013,329.53128 Low-income countries,,2014,312.78476 Low-income countries,,2015,321.6885 Low-income countries,,2016,326.52213 Low-income countries,,2017,329.20953 -Low-income countries,,2018,319.68732 -Low-income countries,,2019,316.64798 -Low-income countries,,2020,289.27777 -Low-income countries,,2021,282.1609 -Low-income countries,,2022,288.4052 -Lower-middle-income countries,,2000,611.1714 -Lower-middle-income countries,,2001,614.7167 -Lower-middle-income countries,,2002,615.77386 -Lower-middle-income countries,,2003,611.41736 -Lower-middle-income countries,,2004,598.90314 -Lower-middle-income countries,,2005,599.94684 -Lower-middle-income countries,,2006,600.4606 -Lower-middle-income countries,,2007,604.0067 -Lower-middle-income countries,,2008,615.2383 -Lower-middle-income countries,,2009,621.8146 -Lower-middle-income countries,,2010,617.89954 -Lower-middle-income countries,,2011,613.03644 -Lower-middle-income countries,,2012,620.497 -Lower-middle-income countries,,2013,619.0321 -Lower-middle-income countries,,2014,627.0608 -Lower-middle-income countries,,2015,634.8372 -Lower-middle-income countries,,2016,640.52563 -Lower-middle-income countries,,2017,629.9125 -Lower-middle-income countries,,2018,632.36334 -Lower-middle-income countries,,2019,622.46124 -Lower-middle-income countries,,2020,615.0575 -Lower-middle-income countries,,2021,616.47144 -Lower-middle-income countries,,2022,616.12933 -Luxembourg,LUX,1990,564.5161 -Luxembourg,LUX,1991,557.1428 -Luxembourg,LUX,1992,560.6061 -Luxembourg,LUX,1993,582.08954 -Luxembourg,LUX,1994,508.4746 -Luxembourg,LUX,1995,408.16327 -Luxembourg,LUX,1996,400.00003 -Luxembourg,LUX,1997,365.85367 -Luxembourg,LUX,1998,216.21622 -Luxembourg,LUX,1999,250.00002 -Luxembourg,LUX,2000,225.00002 +Low-income countries,,2018,334.41437 +Low-income countries,,2019,327.9304 +Low-income countries,,2020,302.78467 +Low-income countries,,2021,291.4869 +Low-income countries,,2022,273.10455 +Lower-middle-income countries,,2000,638.6798 +Lower-middle-income countries,,2001,640.9345 +Lower-middle-income countries,,2002,641.7841 +Lower-middle-income countries,,2003,636.46857 +Lower-middle-income countries,,2004,625.31744 +Lower-middle-income countries,,2005,625.97675 +Lower-middle-income countries,,2006,623.28906 +Lower-middle-income countries,,2007,624.4456 +Lower-middle-income countries,,2008,632.91266 +Lower-middle-income countries,,2009,638.0642 +Lower-middle-income countries,,2010,634.08215 +Lower-middle-income countries,,2011,626.46606 +Lower-middle-income countries,,2012,634.3922 +Lower-middle-income countries,,2013,632.4382 +Lower-middle-income countries,,2014,642.1003 +Lower-middle-income countries,,2015,651.0639 +Lower-middle-income countries,,2016,656.00507 +Lower-middle-income countries,,2017,645.78735 +Lower-middle-income countries,,2018,646.3905 +Lower-middle-income countries,,2019,639.54004 +Lower-middle-income countries,,2020,627.2164 +Lower-middle-income countries,,2021,624.2525 +Lower-middle-income countries,,2022,615.04095 +Lower-middle-income countries,,2023,630.8106 +Luxembourg,LUX,2000,225 Luxembourg,LUX,2001,287.35632 Luxembourg,LUX,2002,317.85715 Luxembourg,LUX,2003,318.99643 @@ -3175,28 +3135,29 @@ Luxembourg,LUX,2007,310.34482 Luxembourg,LUX,2008,308.82352 Luxembourg,LUX,2009,311.1111 Luxembourg,LUX,2010,311.7284 -Luxembourg,LUX,2011,314.39395 +Luxembourg,LUX,2011,314.39392 Luxembourg,LUX,2012,304.3478 Luxembourg,LUX,2013,288.04346 Luxembourg,LUX,2014,287.95813 -Luxembourg,LUX,2015,268.65674 -Luxembourg,LUX,2016,215.18988 +Luxembourg,LUX,2015,268.6567 +Luxembourg,LUX,2016,215.18987 Luxembourg,LUX,2017,188.8889 Luxembourg,LUX,2018,187.50002 Luxembourg,LUX,2019,168.2243 Luxembourg,LUX,2020,163.93442 Luxembourg,LUX,2021,162.60162 -Luxembourg,LUX,2022,144.06781 -Luxembourg,LUX,2023,105.26315 +Luxembourg,LUX,2022,141.66666 +Luxembourg,LUX,2023,122.30216 +Luxembourg,LUX,2024,134.61539 Macao,MAC,2000,656.0509 -Macao,MAC,2001,636.9427 -Macao,MAC,2002,641.1765 +Macao,MAC,2001,636.9426 +Macao,MAC,2002,641.17645 Macao,MAC,2003,622.2222 Macao,MAC,2004,624.3655 Macao,MAC,2005,620.68964 -Macao,MAC,2006,616.7664 +Macao,MAC,2006,616.7665 Macao,MAC,2007,611.8421 -Macao,MAC,2008,561.9835 +Macao,MAC,2008,561.98346 Macao,MAC,2009,582.1918 Macao,MAC,2010,527.7778 Macao,MAC,2011,528.0899 @@ -3206,138 +3167,132 @@ Macao,MAC,2014,468.75003 Macao,MAC,2015,557.8947 Macao,MAC,2016,565.65656 Macao,MAC,2017,564.62585 -Macao,MAC,2018,507.69232 -Macao,MAC,2019,499.99997 +Macao,MAC,2018,507.69235 +Macao,MAC,2019,500 Macao,MAC,2020,482.14288 Macao,MAC,2021,460 Macao,MAC,2022,448.97958 Madagascar,MDG,2000,244.18604 Madagascar,MDG,2001,261.36365 Madagascar,MDG,2002,244.18604 -Madagascar,MDG,2003,249.99998 +Madagascar,MDG,2003,250 Madagascar,MDG,2004,266.6667 Madagascar,MDG,2005,273.5849 Madagascar,MDG,2006,271.028 Madagascar,MDG,2007,250 Madagascar,MDG,2008,288.13562 -Madagascar,MDG,2009,268.90756 +Madagascar,MDG,2009,268.90753 Madagascar,MDG,2010,315.3846 Madagascar,MDG,2011,367.34695 Madagascar,MDG,2012,358.97437 Madagascar,MDG,2013,351.51514 -Madagascar,MDG,2014,331.39536 -Madagascar,MDG,2015,324.0223 +Madagascar,MDG,2014,331.39532 +Madagascar,MDG,2015,324.02234 Madagascar,MDG,2016,357.89474 Madagascar,MDG,2017,409.0909 -Madagascar,MDG,2018,363.63635 -Madagascar,MDG,2019,389.90826 -Madagascar,MDG,2020,407.4074 -Madagascar,MDG,2021,436.68124 -Madagascar,MDG,2022,436.44064 -Malawi,MWI,2000,99.17355 +Madagascar,MDG,2018,424.36972 +Madagascar,MDG,2019,449.3927 +Madagascar,MDG,2020,416.66663 +Madagascar,MDG,2021,486.16602 +Madagascar,MDG,2022,477.2727 +Malawi,MWI,2000,99.173546 Malawi,MWI,2001,90.16393 -Malawi,MWI,2002,101.56249 -Malawi,MWI,2003,91.60305 -Malawi,MWI,2004,95.2381 +Malawi,MWI,2002,101.5625 +Malawi,MWI,2003,91.60306 +Malawi,MWI,2004,95.23809 Malawi,MWI,2005,98.03922 Malawi,MWI,2006,95.541405 -Malawi,MWI,2007,88.6076 -Malawi,MWI,2008,76.47059 +Malawi,MWI,2007,88.60759 +Malawi,MWI,2008,76.47058 Malawi,MWI,2009,77.84431 -Malawi,MWI,2010,67.70833 +Malawi,MWI,2010,67.708336 Malawi,MWI,2011,65.989845 Malawi,MWI,2012,68.29269 Malawi,MWI,2013,68.29269 Malawi,MWI,2014,64.51613 Malawi,MWI,2015,69.12443 Malawi,MWI,2016,69.76744 -Malawi,MWI,2017,26.178009 -Malawi,MWI,2018,145.25139 -Malawi,MWI,2019,62.89308 -Malawi,MWI,2020,39.215687 -Malawi,MWI,2021,69.230774 -Malawi,MWI,2022,66.66667 -Malaysia,MYS,2000,581.8311 +Malawi,MWI,2017,26.17801 +Malawi,MWI,2018,143.64641 +Malawi,MWI,2019,55.555557 +Malawi,MWI,2020,34.285713 +Malawi,MWI,2021,54.64481 +Malawi,MWI,2022,54.64481 +Malaysia,MYS,2000,581.8312 Malaysia,MYS,2001,600.0264 -Malaysia,MYS,2002,618.2762 +Malaysia,MYS,2002,618.27625 Malaysia,MYS,2003,631.1368 Malaysia,MYS,2004,649.8022 Malaysia,MYS,2005,652.4557 Malaysia,MYS,2006,652.65894 Malaysia,MYS,2007,662.27545 -Malaysia,MYS,2008,651.05853 +Malaysia,MYS,2008,651.0586 Malaysia,MYS,2009,666.60925 -Malaysia,MYS,2010,688.9795 -Malaysia,MYS,2011,685.685 -Malaysia,MYS,2012,682.7267 +Malaysia,MYS,2010,688.97955 +Malaysia,MYS,2011,685.68506 +Malaysia,MYS,2012,682.72675 Malaysia,MYS,2013,666.0282 -Malaysia,MYS,2014,662.30414 -Malaysia,MYS,2015,669.17596 +Malaysia,MYS,2014,662.3042 +Malaysia,MYS,2015,669.1759 Malaysia,MYS,2016,649.28186 Malaysia,MYS,2017,620.5802 Malaysia,MYS,2018,630.32007 -Malaysia,MYS,2019,626.27454 -Malaysia,MYS,2020,626.9677 -Malaysia,MYS,2021,607.0917 -Malaysia,MYS,2022,605.83136 +Malaysia,MYS,2019,626.0504 +Malaysia,MYS,2020,633.6992 +Malaysia,MYS,2021,611.4377 +Malaysia,MYS,2022,604.0407 +Malaysia,MYS,2023,607.88074 +Malaysia,MYS,2024,604.4294 Maldives,MDV,2000,666.6666 Maldives,MDV,2001,687.5 Maldives,MDV,2002,684.2105 Maldives,MDV,2003,636.36365 Maldives,MDV,2004,666.6667 -Maldives,MDV,2005,666.66675 -Maldives,MDV,2006,645.16125 +Maldives,MDV,2005,666.6667 +Maldives,MDV,2006,645.1613 Maldives,MDV,2007,657.1429 Maldives,MDV,2008,658.5366 -Maldives,MDV,2009,666.6666 +Maldives,MDV,2009,666.6667 Maldives,MDV,2010,657.1429 -Maldives,MDV,2011,657.8947 -Maldives,MDV,2012,650 +Maldives,MDV,2011,657.8948 +Maldives,MDV,2012,649.99994 Maldives,MDV,2013,666.6667 Maldives,MDV,2014,648.14813 -Maldives,MDV,2015,645.16125 -Maldives,MDV,2016,641.7911 -Maldives,MDV,2017,643.83563 +Maldives,MDV,2015,645.1613 +Maldives,MDV,2016,641.791 +Maldives,MDV,2017,643.8356 Maldives,MDV,2018,632.9114 -Maldives,MDV,2019,629.6296 -Maldives,MDV,2020,621.9512 +Maldives,MDV,2019,629.62964 +Maldives,MDV,2020,621.95123 Maldives,MDV,2021,619.0476 -Maldives,MDV,2022,611.7647 +Maldives,MDV,2022,611.76465 +Maldives,MDV,2023,611.76465 Mali,MLI,2000,500 Mali,MLI,2001,504.67288 -Mali,MLI,2002,504.50452 -Mali,MLI,2003,513.51355 +Mali,MLI,2002,504.5045 +Mali,MLI,2003,513.5135 Mali,MLI,2004,515.873 Mali,MLI,2005,449.66443 Mali,MLI,2006,445.85986 -Mali,MLI,2007,455.69623 -Mali,MLI,2008,441.98895 -Mali,MLI,2009,371.79483 +Mali,MLI,2007,455.6962 +Mali,MLI,2008,441.98898 +Mali,MLI,2009,371.79486 Mali,MLI,2010,351.69492 -Mali,MLI,2011,305.55557 +Mali,MLI,2011,305.55554 Mali,MLI,2012,298.7013 Mali,MLI,2013,318.1818 Mali,MLI,2014,290.83667 -Mali,MLI,2015,358.56577 +Mali,MLI,2015,358.56573 Mali,MLI,2016,331.12582 Mali,MLI,2017,372.09302 -Mali,MLI,2018,387.57394 -Mali,MLI,2019,448.71793 -Mali,MLI,2020,451.10406 -Mali,MLI,2021,446.4832 -Mali,MLI,2022,407.99997 -Malta,MLT,1990,792.7928 -Malta,MLT,1991,760.5634 -Malta,MLT,1992,751.67786 -Malta,MLT,1993,753.3334 -Malta,MLT,1994,720.77924 -Malta,MLT,1995,668.7117 -Malta,MLT,1996,656.6265 -Malta,MLT,1997,656.80475 -Malta,MLT,1998,656.97675 -Malta,MLT,1999,659.45953 +Mali,MLI,2018,404.43216 +Mali,MLI,2019,417.32285 +Mali,MLI,2020,401.01523 +Mali,MLI,2021,416.24365 +Mali,MLI,2022,391.70508 +Mali,MLI,2023,394.49542 Malta,MLT,2000,656.25 -Malta,MLT,2001,659.7938 +Malta,MLT,2001,659.79376 Malta,MLT,2002,658.5366 Malta,MLT,2003,656.25 Malta,MLT,2004,657.65765 @@ -3346,185 +3301,192 @@ Malta,MLT,2006,659.29205 Malta,MLT,2007,656.5217 Malta,MLT,2008,658.00867 Malta,MLT,2009,658.98615 -Malta,MLT,2010,658.76776 +Malta,MLT,2010,658.7678 Malta,MLT,2011,658.98615 Malta,MLT,2012,647.8261 Malta,MLT,2013,646.0177 -Malta,MLT,2014,635.5556 +Malta,MLT,2014,635.55554 Malta,MLT,2015,607.6923 Malta,MLT,2016,558.1395 Malta,MLT,2017,527.27277 Malta,MLT,2018,512.69037 Malta,MLT,2019,516.90826 Malta,MLT,2020,506.97675 -Malta,MLT,2021,504.50452 -Malta,MLT,2022,499.99997 -Malta,MLT,2023,459.14395 +Malta,MLT,2021,504.5045 +Malta,MLT,2022,500 +Malta,MLT,2023,493.61703 +Malta,MLT,2024,484.1629 Martinique,MTQ,2000,657.8948 Martinique,MTQ,2001,656.4886 Martinique,MTQ,2002,651.8518 Martinique,MTQ,2003,650 -Martinique,MTQ,2004,655.17236 -Martinique,MTQ,2005,655.17236 -Martinique,MTQ,2006,646.6667 -Martinique,MTQ,2007,644.2953 -Martinique,MTQ,2008,642.8571 -Martinique,MTQ,2009,641.02563 -Martinique,MTQ,2010,641.9753 -Martinique,MTQ,2011,639.2405 -Martinique,MTQ,2012,618.75 -Martinique,MTQ,2013,618.75 -Martinique,MTQ,2014,615.3846 -Martinique,MTQ,2015,613.9241 -Martinique,MTQ,2016,612.5 +Martinique,MTQ,2004,652.7778 +Martinique,MTQ,2005,652.7778 +Martinique,MTQ,2006,653.0612 +Martinique,MTQ,2007,648.6486 +Martinique,MTQ,2008,647.05884 +Martinique,MTQ,2009,642.8572 +Martinique,MTQ,2010,645.9627 +Martinique,MTQ,2011,636.9426 +Martinique,MTQ,2012,622.6415 +Martinique,MTQ,2013,620.2532 +Martinique,MTQ,2014,619.35486 +Martinique,MTQ,2015,621.7949 +Martinique,MTQ,2016,620.2532 Martinique,MTQ,2017,621.7949 Martinique,MTQ,2018,590.9091 -Martinique,MTQ,2019,522.5806 +Martinique,MTQ,2019,522.5807 Martinique,MTQ,2020,540 Martinique,MTQ,2021,523.17883 +Martinique,MTQ,2022,516.7785 Mauritania,MRT,2000,619.0476 Mauritania,MRT,2001,617.02124 Mauritania,MRT,2002,622.64154 -Mauritania,MRT,2003,600.00006 -Mauritania,MRT,2004,611.11115 +Mauritania,MRT,2003,600 +Mauritania,MRT,2004,611.1111 Mauritania,MRT,2005,603.7736 Mauritania,MRT,2006,612.9032 Mauritania,MRT,2007,609.375 Mauritania,MRT,2008,597.2222 Mauritania,MRT,2009,558.4416 Mauritania,MRT,2010,564.7059 -Mauritania,MRT,2011,563.21844 -Mauritania,MRT,2012,562.5 +Mauritania,MRT,2011,563.2184 +Mauritania,MRT,2012,562.50006 Mauritania,MRT,2013,520.40814 -Mauritania,MRT,2014,514.56305 -Mauritania,MRT,2015,474.13794 +Mauritania,MRT,2014,514.5631 +Mauritania,MRT,2015,474.13797 Mauritania,MRT,2016,491.37933 -Mauritania,MRT,2017,496.29633 -Mauritania,MRT,2018,468.96555 -Mauritania,MRT,2019,467.94873 -Mauritania,MRT,2020,469.51218 -Mauritania,MRT,2021,460.1227 -Mauritania,MRT,2022,464.7059 +Mauritania,MRT,2017,496.2963 +Mauritania,MRT,2018,465.75342 +Mauritania,MRT,2019,464.96814 +Mauritania,MRT,2020,472.39264 +Mauritania,MRT,2021,474.68353 +Mauritania,MRT,2022,481.70734 Mauritius,MUS,2000,559.322 Mauritius,MUS,2001,567.7084 -Mauritius,MUS,2002,581.6327 +Mauritius,MUS,2002,581.6326 Mauritius,MUS,2003,574.1627 -Mauritius,MUS,2004,564.8149 +Mauritius,MUS,2004,564.8148 Mauritius,MUS,2005,592.92035 -Mauritius,MUS,2006,622.88135 +Mauritius,MUS,2006,622.8814 Mauritius,MUS,2007,634.1463 Mauritius,MUS,2008,636.71875 Mauritius,MUS,2009,626.45917 -Mauritius,MUS,2010,620.81775 +Mauritius,MUS,2010,620.8178 Mauritius,MUS,2011,648.3516 -Mauritius,MUS,2012,645.1613 +Mauritius,MUS,2012,645.16125 Mauritius,MUS,2013,646.85315 Mauritius,MUS,2014,642.8571 -Mauritius,MUS,2015,622.07355 +Mauritius,MUS,2015,622.0736 Mauritius,MUS,2016,632.78687 Mauritius,MUS,2017,643.08685 -Mauritius,MUS,2018,634.61536 -Mauritius,MUS,2019,619.195 +Mauritius,MUS,2018,634.6154 +Mauritius,MUS,2019,619.19507 Mauritius,MUS,2020,613.7931 Mauritius,MUS,2021,628.7625 -Mauritius,MUS,2022,632.47864 +Mauritius,MUS,2022,615.38464 +Mauritius,MUS,2023,633.0275 Mexico,MEX,2000,525.39044 Mexico,MEX,2001,543.9064 Mexico,MEX,2002,553.4233 -Mexico,MEX,2003,570.16455 +Mexico,MEX,2003,570.1645 Mexico,MEX,2004,553.93176 Mexico,MEX,2005,555.8334 Mexico,MEX,2006,546.80975 Mexico,MEX,2007,554.2049 Mexico,MEX,2008,522.0394 Mexico,MEX,2009,554.07825 -Mexico,MEX,2010,543.472 -Mexico,MEX,2011,540.97345 -Mexico,MEX,2012,551.3158 -Mexico,MEX,2013,547.59015 -Mexico,MEX,2014,526.9682 -Mexico,MEX,2015,534.87994 -Mexico,MEX,2016,537.0649 -Mexico,MEX,2017,531.9517 -Mexico,MEX,2018,523.4469 -Mexico,MEX,2019,525.7277 -Mexico,MEX,2020,488.45703 -Mexico,MEX,2021,461.72723 -Mexico,MEX,2022,475.3588 -Mexico,MEX,2023,507.24512 +Mexico,MEX,2010,541.9312 +Mexico,MEX,2011,539.7912 +Mexico,MEX,2012,550.2547 +Mexico,MEX,2013,546.8161 +Mexico,MEX,2014,525.5992 +Mexico,MEX,2015,533.8961 +Mexico,MEX,2016,537.5995 +Mexico,MEX,2017,533.68384 +Mexico,MEX,2018,523.9199 +Mexico,MEX,2019,525.72766 +Mexico,MEX,2020,486.9221 +Mexico,MEX,2021,461.39392 +Mexico,MEX,2022,472.58563 +Mexico,MEX,2023,492.33505 +Mexico,MEX,2024,484.83173 Middle East (Ember),,2000,685.8462 Middle East (Ember),,2001,688.5609 Middle East (Ember),,2002,686.3636 Middle East (Ember),,2003,684.31903 -Middle East (Ember),,2004,679.6518 -Middle East (Ember),,2005,673.77515 +Middle East (Ember),,2004,679.65173 +Middle East (Ember),,2005,673.7752 Middle East (Ember),,2006,669.49023 Middle East (Ember),,2007,670.69275 Middle East (Ember),,2008,680.4821 Middle East (Ember),,2009,681.1366 -Middle East (Ember),,2010,678.6357 -Middle East (Ember),,2011,676.59076 -Middle East (Ember),,2012,675.1784 -Middle East (Ember),,2013,668.2842 -Middle East (Ember),,2014,672.13385 -Middle East (Ember),,2015,674.1551 -Middle East (Ember),,2016,670.67804 -Middle East (Ember),,2017,671.31604 -Middle East (Ember),,2018,674.1818 -Middle East (Ember),,2019,657.29407 -Middle East (Ember),,2020,660.8495 -Middle East (Ember),,2021,660.4582 -Middle East (Ember),,2022,660.2886 -Middle East (Ember),,2023,657.5647 +Middle East (Ember),,2010,677.75775 +Middle East (Ember),,2011,676.1312 +Middle East (Ember),,2012,674.3106 +Middle East (Ember),,2013,667.18567 +Middle East (Ember),,2014,670.7617 +Middle East (Ember),,2015,673.3079 +Middle East (Ember),,2016,669.1949 +Middle East (Ember),,2017,670.39545 +Middle East (Ember),,2018,674.143 +Middle East (Ember),,2019,658.3865 +Middle East (Ember),,2020,661.9074 +Middle East (Ember),,2021,662.5883 +Middle East (Ember),,2022,654.4757 +Middle East (Ember),,2023,641.2225 +Middle East (Ember),,2024,636.3722 Moldova,MDA,2000,662.3853 Moldova,MDA,2001,670.0855 Moldova,MDA,2002,668.43976 Moldova,MDA,2003,668.9775 -Moldova,MDA,2004,670.6897 +Moldova,MDA,2004,670.68964 Moldova,MDA,2005,671.69183 Moldova,MDA,2006,671.0526 Moldova,MDA,2007,671.20953 -Moldova,MDA,2008,668.88513 +Moldova,MDA,2008,668.8852 Moldova,MDA,2009,672.05164 -Moldova,MDA,2010,666.1211 +Moldova,MDA,2010,666.12115 Moldova,MDA,2011,669.5501 Moldova,MDA,2012,679.8623 -Moldova,MDA,2013,665.1885 +Moldova,MDA,2013,665.1884 Moldova,MDA,2014,670.39105 Moldova,MDA,2015,682.1192 Moldova,MDA,2016,683.7607 -Moldova,MDA,2017,669.35486 +Moldova,MDA,2017,669.3548 Moldova,MDA,2018,672.8624 Moldova,MDA,2019,669.59576 Moldova,MDA,2020,673.13916 Moldova,MDA,2021,668.16815 -Moldova,MDA,2022,648.4963 -Moldova,MDA,2023,643.4635 +Moldova,MDA,2022,647.27954 +Moldova,MDA,2023,625.74255 +Moldova,MDA,2024,629.55853 Mongolia,MNG,2000,872.41376 Mongolia,MNG,2001,872.05383 -Mongolia,MNG,2002,873.7865 +Mongolia,MNG,2002,873.78644 Mongolia,MNG,2003,871.3827 -Mongolia,MNG,2004,871.95123 -Mongolia,MNG,2005,870.58826 +Mongolia,MNG,2004,871.9512 +Mongolia,MNG,2005,870.5882 Mongolia,MNG,2006,872.52124 Mongolia,MNG,2007,867.9245 -Mongolia,MNG,2008,867.50006 +Mongolia,MNG,2008,867.5 Mongolia,MNG,2009,861.38617 -Mongolia,MNG,2010,856.4815 +Mongolia,MNG,2010,856.48145 Mongolia,MNG,2011,854.6256 -Mongolia,MNG,2012,852.3908 +Mongolia,MNG,2012,852.39087 Mongolia,MNG,2013,844.62146 Mongolia,MNG,2014,832.71375 -Mongolia,MNG,2015,831.216 +Mongolia,MNG,2015,831.21594 Mongolia,MNG,2016,830.6878 Mongolia,MNG,2017,829.1874 -Mongolia,MNG,2018,808.2191 +Mongolia,MNG,2018,808.2192 Mongolia,MNG,2019,779.5389 -Mongolia,MNG,2020,777.10846 +Mongolia,MNG,2020,777.1084 Mongolia,MNG,2021,770.1613 -Mongolia,MNG,2022,771.79486 -Mongolia,MNG,2023,775.30865 -Montenegro,MNE,2005,369.33795 +Mongolia,MNG,2022,781.9355 +Mongolia,MNG,2023,785.0822 +Mongolia,MNG,2024,784.0096 +Montenegro,MNE,2005,369.33798 Montenegro,MNE,2006,427.11862 Montenegro,MNE,2007,420.56073 Montenegro,MNE,2008,477.03183 @@ -3532,17 +3494,18 @@ Montenegro,MNE,2009,271.73914 Montenegro,MNE,2010,338.30847 Montenegro,MNE,2011,566.0377 Montenegro,MNE,2012,501.7544 -Montenegro,MNE,2013,385.78677 +Montenegro,MNE,2013,385.7868 Montenegro,MNE,2014,466.87695 Montenegro,MNE,2015,523.3334 -Montenegro,MNE,2016,433.12103 +Montenegro,MNE,2016,433.121 Montenegro,MNE,2017,564.5161 Montenegro,MNE,2018,426.3158 Montenegro,MNE,2019,454.81046 -Montenegro,MNE,2020,495.57516 +Montenegro,MNE,2020,495.5752 Montenegro,MNE,2021,400.53052 Montenegro,MNE,2022,483.3837 -Montenegro,MNE,2023,417.07318 +Montenegro,MNE,2023,407.58298 +Montenegro,MNE,2024,413.5135 Montserrat,MSR,2000,1000 Montserrat,MSR,2001,1000 Montserrat,MSR,2002,1000 @@ -3566,10 +3529,10 @@ Montserrat,MSR,2019,1000 Montserrat,MSR,2020,1000 Montserrat,MSR,2021,1000 Montserrat,MSR,2022,1000 -Morocco,MAR,2000,759.124 +Morocco,MAR,2000,759.1241 Morocco,MAR,2001,769.2308 -Morocco,MAR,2002,765.56287 -Morocco,MAR,2003,735.00616 +Morocco,MAR,2002,765.5629 +Morocco,MAR,2003,735.0061 Morocco,MAR,2004,732.68463 Morocco,MAR,2005,717.03015 Morocco,MAR,2006,708.6059 @@ -3577,25 +3540,26 @@ Morocco,MAR,2007,707.4657 Morocco,MAR,2008,695.82306 Morocco,MAR,2009,635.4515 Morocco,MAR,2010,610.50323 -Morocco,MAR,2011,652.7093 +Morocco,MAR,2011,652.70935 Morocco,MAR,2012,648.4346 Morocco,MAR,2013,612.47217 Morocco,MAR,2014,651.35333 Morocco,MAR,2015,639.91974 -Morocco,MAR,2016,633.6249 -Morocco,MAR,2017,637.44525 -Morocco,MAR,2018,621.5278 +Morocco,MAR,2016,633.6248 +Morocco,MAR,2017,637.4452 +Morocco,MAR,2018,621.5277 Morocco,MAR,2019,631.74835 Morocco,MAR,2020,645.6185 -Morocco,MAR,2021,639.95154 +Morocco,MAR,2021,639.9516 Morocco,MAR,2022,662.6419 -Morocco,MAR,2023,630.01416 -Mozambique,MOZ,2000,25.07837 +Morocco,MAR,2023,616.81586 +Morocco,MAR,2024,577.6511 +Mozambique,MOZ,2000,25.078371 Mozambique,MOZ,2001,26.360544 -Mozambique,MOZ,2002,25.437202 -Mozambique,MOZ,2003,25.949955 -Mozambique,MOZ,2004,26.724138 -Mozambique,MOZ,2005,25.095057 +Mozambique,MOZ,2002,25.4372 +Mozambique,MOZ,2003,25.949953 +Mozambique,MOZ,2004,26.724136 +Mozambique,MOZ,2005,25.095058 Mozambique,MOZ,2006,25.359835 Mozambique,MOZ,2007,24.512884 Mozambique,MOZ,2008,24.716099 @@ -3604,62 +3568,65 @@ Mozambique,MOZ,2010,24.595081 Mozambique,MOZ,2011,27.727547 Mozambique,MOZ,2012,28.05611 Mozambique,MOZ,2013,32.952255 -Mozambique,MOZ,2014,32.31707 +Mozambique,MOZ,2014,32.317074 Mozambique,MOZ,2015,68.008705 Mozambique,MOZ,2016,132.55566 Mozambique,MOZ,2017,147.95918 Mozambique,MOZ,2018,139.43991 Mozambique,MOZ,2019,154.29173 Mozambique,MOZ,2020,128.28601 -Mozambique,MOZ,2021,131.49521 -Mozambique,MOZ,2022,135.64668 +Mozambique,MOZ,2021,131.49522 +Mozambique,MOZ,2022,127.81187 +Mozambique,MOZ,2023,127.81187 Myanmar,MMR,2000,568.6275 Myanmar,MMR,2001,558.8865 Myanmar,MMR,2002,550.29584 -Myanmar,MMR,2003,581.1808 -Myanmar,MMR,2004,543.67194 -Myanmar,MMR,2005,478.40536 +Myanmar,MMR,2003,581.18085 +Myanmar,MMR,2004,543.672 +Myanmar,MMR,2005,478.40533 Myanmar,MMR,2006,438.51135 Myanmar,MMR,2007,412.5 Myanmar,MMR,2008,371.60123 Myanmar,MMR,2009,243.90245 -Myanmar,MMR,2010,277.26218 -Myanmar,MMR,2011,275.1678 +Myanmar,MMR,2010,277.2622 +Myanmar,MMR,2011,275.16776 Myanmar,MMR,2012,284.0502 Myanmar,MMR,2013,276.08347 Myanmar,MMR,2014,366.22653 -Myanmar,MMR,2015,398.1481 +Myanmar,MMR,2015,398.14813 Myanmar,MMR,2016,439.84543 Myanmar,MMR,2017,451.0456 -Myanmar,MMR,2018,405.26804 -Myanmar,MMR,2019,419.25467 -Myanmar,MMR,2020,487.22043 -Myanmar,MMR,2021,484.24216 -Myanmar,MMR,2022,483.57037 -Myanmar,MMR,2023,398.89804 +Myanmar,MMR,2018,405.26807 +Myanmar,MMR,2019,419.2547 +Myanmar,MMR,2020,487.2204 +Myanmar,MMR,2021,484.4845 +Myanmar,MMR,2022,563.871 +Myanmar,MMR,2023,588.9451 +Myanmar,MMR,2024,569.6921 Namibia,NAM,2000,41.666664 -Namibia,NAM,2001,24.793388 +Namibia,NAM,2001,24.793386 Namibia,NAM,2002,27.972029 Namibia,NAM,2003,27.972029 -Namibia,NAM,2004,28.985508 -Namibia,NAM,2005,41.42012 +Namibia,NAM,2004,28.985506 +Namibia,NAM,2005,41.420116 Namibia,NAM,2006,80.74534 Namibia,NAM,2007,44.585987 Namibia,NAM,2008,121.019104 Namibia,NAM,2009,66.666664 Namibia,NAM,2010,53.435116 -Namibia,NAM,2011,48.275864 +Namibia,NAM,2011,48.27586 Namibia,NAM,2012,47.90419 Namibia,NAM,2013,51.851852 -Namibia,NAM,2014,32.679737 -Namibia,NAM,2015,44.3038 -Namibia,NAM,2016,54.054054 +Namibia,NAM,2014,32.67974 +Namibia,NAM,2015,44.303795 +Namibia,NAM,2016,54.05405 Namibia,NAM,2017,62.146893 -Namibia,NAM,2018,42.55319 -Namibia,NAM,2019,57.971016 -Namibia,NAM,2020,59.999996 +Namibia,NAM,2018,42.553192 +Namibia,NAM,2019,57.971012 +Namibia,NAM,2020,60 Namibia,NAM,2021,46.357616 -Namibia,NAM,2022,59.25926 +Namibia,NAM,2022,45.454544 +Namibia,NAM,2023,47.61905 Nauru,NRU,2000,666.6667 Nauru,NRU,2001,666.6667 Nauru,NRU,2002,666.6667 @@ -3683,39 +3650,30 @@ Nauru,NRU,2019,750 Nauru,NRU,2020,750 Nauru,NRU,2021,750 Nauru,NRU,2022,750 +Nauru,NRU,2023,750 Nepal,NPL,2000,36.363636 Nepal,NPL,2001,27.027027 Nepal,NPL,2002,23.809525 Nepal,NPL,2003,22.123894 Nepal,NPL,2004,29.288702 -Nepal,NPL,2005,27.888449 +Nepal,NPL,2005,27.888447 Nepal,NPL,2006,25.735294 -Nepal,NPL,2007,28.880867 +Nepal,NPL,2007,28.880865 Nepal,NPL,2008,28.673836 Nepal,NPL,2009,25.974026 Nepal,NPL,2010,25.157232 -Nepal,NPL,2011,23.121387 -Nepal,NPL,2012,25.495754 -Nepal,NPL,2013,25.714289 +Nepal,NPL,2011,23.121386 +Nepal,NPL,2012,25.495752 +Nepal,NPL,2013,25.714287 Nepal,NPL,2014,23.684212 -Nepal,NPL,2015,23.121387 +Nepal,NPL,2015,23.121386 Nepal,NPL,2016,23.923447 Nepal,NPL,2017,24.017467 Nepal,NPL,2018,22.727272 -Nepal,NPL,2019,22.801304 -Nepal,NPL,2020,22.580645 -Nepal,NPL,2021,24.819029 -Nepal,NPL,2022,24.43992 -Netherlands,NLD,1990,590.8062 -Netherlands,NLD,1991,569.1068 -Netherlands,NLD,1992,559.0718 -Netherlands,NLD,1993,550.4744 -Netherlands,NLD,1994,564.2729 -Netherlands,NLD,1995,581.5994 -Netherlands,NLD,1996,561.19904 -Netherlands,NLD,1997,559.107 -Netherlands,NLD,1998,557.1093 -Netherlands,NLD,1999,535.1104 +Nepal,NPL,2019,22.764227 +Nepal,NPL,2020,22.653723 +Nepal,NPL,2021,23.883698 +Nepal,NPL,2022,23.364487 Netherlands,NLD,2000,542.3575 Netherlands,NLD,2001,543.0866 Netherlands,NLD,2002,540.78613 @@ -3723,475 +3681,471 @@ Netherlands,NLD,2003,542.7883 Netherlands,NLD,2004,524.64435 Netherlands,NLD,2005,513.54913 Netherlands,NLD,2006,513.2281 -Netherlands,NLD,2007,511.5741 +Netherlands,NLD,2007,511.57407 Netherlands,NLD,2008,496.09265 Netherlands,NLD,2009,488.29553 -Netherlands,NLD,2010,479.59354 +Netherlands,NLD,2010,479.59357 Netherlands,NLD,2011,473.7216 -Netherlands,NLD,2012,496.2355 -Netherlands,NLD,2013,502.3867 -Netherlands,NLD,2014,523.4084 -Netherlands,NLD,2015,553.0783 +Netherlands,NLD,2012,496.23547 +Netherlands,NLD,2013,502.38666 +Netherlands,NLD,2014,523.40845 +Netherlands,NLD,2015,553.07825 Netherlands,NLD,2016,525.18427 Netherlands,NLD,2017,489.91727 -Netherlands,NLD,2018,467.1088 +Netherlands,NLD,2018,467.10876 Netherlands,NLD,2019,409.99585 Netherlands,NLD,2020,342.89932 -Netherlands,NLD,2021,352.35175 -Netherlands,NLD,2022,324.99167 -Netherlands,NLD,2023,267.62177 -New Caledonia,NCL,2000,512.19507 +Netherlands,NLD,2021,352.23508 +Netherlands,NLD,2022,326.07062 +Netherlands,NLD,2023,268.68045 +Netherlands,NLD,2024,253.31152 +New Caledonia,NCL,2000,512.1951 New Caledonia,NCL,2001,552.32556 New Caledonia,NCL,2002,582.8571 New Caledonia,NCL,2003,571.4286 New Caledonia,NCL,2004,562.87427 -New Caledonia,NCL,2005,563.8297 -New Caledonia,NCL,2006,582.88776 +New Caledonia,NCL,2005,563.8298 +New Caledonia,NCL,2006,582.8877 New Caledonia,NCL,2007,544.04144 -New Caledonia,NCL,2008,510.75266 +New Caledonia,NCL,2008,510.7527 New Caledonia,NCL,2009,546.3917 -New Caledonia,NCL,2010,636.79254 +New Caledonia,NCL,2010,636.7925 New Caledonia,NCL,2011,606.1947 -New Caledonia,NCL,2012,589.5197 -New Caledonia,NCL,2013,593.62555 -New Caledonia,NCL,2014,651.16284 -New Caledonia,NCL,2015,641.37933 +New Caledonia,NCL,2012,589.51965 +New Caledonia,NCL,2013,593.6255 +New Caledonia,NCL,2014,651.1628 +New Caledonia,NCL,2015,641.3793 New Caledonia,NCL,2016,691.0828 New Caledonia,NCL,2017,674.9226 -New Caledonia,NCL,2018,685.71436 -New Caledonia,NCL,2019,671.55426 -New Caledonia,NCL,2020,698.41266 -New Caledonia,NCL,2021,661.7647 -New Caledonia,NCL,2022,660.5839 +New Caledonia,NCL,2018,685.7143 +New Caledonia,NCL,2019,687.6877 +New Caledonia,NCL,2020,679.0123 +New Caledonia,NCL,2021,618.55664 +New Caledonia,NCL,2022,585.7605 New Zealand,NZL,2000,184.7854 -New Zealand,NZL,2001,230.90633 +New Zealand,NZL,2001,230.90631 New Zealand,NZL,2002,198.2094 -New Zealand,NZL,2003,232.35075 -New Zealand,NZL,2004,205.3466 -New Zealand,NZL,2005,255.6497 +New Zealand,NZL,2003,232.35077 +New Zealand,NZL,2004,205.34659 +New Zealand,NZL,2005,255.64972 New Zealand,NZL,2006,250.23235 -New Zealand,NZL,2007,227.7547 +New Zealand,NZL,2007,227.75473 New Zealand,NZL,2008,248.21388 -New Zealand,NZL,2009,197.43886 +New Zealand,NZL,2009,197.43887 New Zealand,NZL,2010,179.8691 -New Zealand,NZL,2011,166.43866 -New Zealand,NZL,2012,199.08572 -New Zealand,NZL,2013,177.51755 +New Zealand,NZL,2011,166.43867 +New Zealand,NZL,2012,199.08571 +New Zealand,NZL,2013,177.51756 New Zealand,NZL,2014,148.88164 New Zealand,NZL,2015,143.2178 New Zealand,NZL,2016,114.716095 New Zealand,NZL,2017,131.92854 -New Zealand,NZL,2018,123.94238 +New Zealand,NZL,2018,123.942375 New Zealand,NZL,2019,134.17143 -New Zealand,NZL,2020,142.56055 -New Zealand,NZL,2021,147.03238 -New Zealand,NZL,2022,110.88481 -New Zealand,NZL,2023,112.75831 +New Zealand,NZL,2020,142.56056 +New Zealand,NZL,2021,145.5798 +New Zealand,NZL,2022,112.836914 +New Zealand,NZL,2023,104.423164 +New Zealand,NZL,2024,120.10797 Nicaragua,NIC,2000,530.7018 Nicaragua,NIC,2001,538.46155 Nicaragua,NIC,2002,513.72546 -Nicaragua,NIC,2003,500.00003 +Nicaragua,NIC,2003,500 Nicaragua,NIC,2004,498.22064 -Nicaragua,NIC,2005,454.5454 -Nicaragua,NIC,2006,488.81784 +Nicaragua,NIC,2005,454.54544 +Nicaragua,NIC,2006,488.81787 Nicaragua,NIC,2007,500 -Nicaragua,NIC,2008,452.38092 -Nicaragua,NIC,2009,484.05792 +Nicaragua,NIC,2008,452.38095 +Nicaragua,NIC,2009,484.05795 Nicaragua,NIC,2010,443.8356 Nicaragua,NIC,2011,461.94226 Nicaragua,NIC,2012,409.42926 -Nicaragua,NIC,2013,347.7218 +Nicaragua,NIC,2013,347.72183 Nicaragua,NIC,2014,338.565 Nicaragua,NIC,2015,361.65576 Nicaragua,NIC,2016,348.58386 Nicaragua,NIC,2017,329.64603 -Nicaragua,NIC,2018,313.97852 +Nicaragua,NIC,2018,313.9785 Nicaragua,NIC,2019,334.06113 -Nicaragua,NIC,2020,250.65962 +Nicaragua,NIC,2020,250.65964 Nicaragua,NIC,2021,261.28265 -Nicaragua,NIC,2022,265.11627 +Nicaragua,NIC,2022,289.35184 +Nicaragua,NIC,2023,288.32953 Niger,NER,2000,857.14294 -Niger,NER,2001,833.3334 -Niger,NER,2002,842.1053 -Niger,NER,2003,850.00006 -Niger,NER,2004,850.00006 +Niger,NER,2001,833.3333 +Niger,NER,2002,842.1052 +Niger,NER,2003,850 +Niger,NER,2004,850 Niger,NER,2005,869.5652 -Niger,NER,2006,863.6363 -Niger,NER,2007,863.6363 -Niger,NER,2008,863.6363 +Niger,NER,2006,863.63635 +Niger,NER,2007,863.63635 +Niger,NER,2008,863.63635 Niger,NER,2009,846.1539 -Niger,NER,2010,827.5862 -Niger,NER,2011,852.9411 +Niger,NER,2010,827.58624 +Niger,NER,2011,852.94116 Niger,NER,2012,820.5128 Niger,NER,2013,795.4545 Niger,NER,2014,800.00006 Niger,NER,2015,740.7407 Niger,NER,2016,728.5714 -Niger,NER,2017,732.1429 -Niger,NER,2018,666.6666 -Niger,NER,2019,671.875 -Niger,NER,2020,681.8182 -Niger,NER,2021,666.6667 -Niger,NER,2022,670.88605 +Niger,NER,2017,732.1428 +Niger,NER,2018,716.98114 +Niger,NER,2019,701.4925 +Niger,NER,2020,716.41785 +Niger,NER,2021,704.2254 +Niger,NER,2022,696.2025 +Niger,NER,2023,687.5 Nigeria,NGA,2000,411.80463 -Nigeria,NGA,2001,411.7647 +Nigeria,NGA,2001,411.76468 Nigeria,NGA,2002,412.06497 -Nigeria,NGA,2003,420.29697 -Nigeria,NGA,2004,442.15726 +Nigeria,NGA,2003,420.297 +Nigeria,NGA,2004,442.15723 Nigeria,NGA,2005,444.586 Nigeria,NGA,2006,481.8339 -Nigeria,NGA,2007,481.94864 -Nigeria,NGA,2008,482.0076 +Nigeria,NGA,2007,481.94867 +Nigeria,NGA,2008,482.00757 Nigeria,NGA,2009,508.33755 -Nigeria,NGA,2010,499.04324 +Nigeria,NGA,2010,499.04327 Nigeria,NGA,2011,515.53253 Nigeria,NGA,2012,527.83575 Nigeria,NGA,2013,535.776 -Nigeria,NGA,2014,547.3978 -Nigeria,NGA,2015,529.6955 -Nigeria,NGA,2016,511.22055 -Nigeria,NGA,2017,500.00003 -Nigeria,NGA,2018,506.4352 -Nigeria,NGA,2019,491.24402 -Nigeria,NGA,2020,513.2592 -Nigeria,NGA,2021,512.75653 -Nigeria,NGA,2022,516.225 -Nigeria,NGA,2023,523.24725 -North America,,2000,526.5698 -North America,,2001,531.02985 -North America,,2002,523.52295 -North America,,2003,530.4878 -North America,,2004,524.6735 -North America,,2005,524.42096 -North America,,2006,517.10956 -North America,,2007,520.4894 -North America,,2008,511.07025 -North America,,2009,491.75668 -North America,,2010,498.3351 -North America,,2011,480.31714 -North America,,2012,465.6571 -North America,,2013,464.1943 -North America,,2014,460.8257 -North America,,2015,442.0847 -North America,,2016,425.5822 -North America,,2017,412.6762 -North America,,2018,408.77774 -North America,,2019,392.3433 -North America,,2020,366.4238 -North America,,2021,375.01917 -North America,,2022,365.84262 -North America (Ember),,2000,526.54095 -North America (Ember),,2001,530.0896 -North America (Ember),,2002,521.5634 -North America (Ember),,2003,528.00885 -North America (Ember),,2004,522.8291 -North America (Ember),,2005,522.4568 -North America (Ember),,2006,514.945 -North America (Ember),,2007,518.17615 -North America (Ember),,2008,510.0643 -North America (Ember),,2009,487.13464 -North America (Ember),,2010,495.2451 -North America (Ember),,2011,475.48752 -North America (Ember),,2012,459.0329 -North America (Ember),,2013,457.72916 -North America (Ember),,2014,455.25595 -North America (Ember),,2015,434.24826 -North America (Ember),,2016,415.99686 -North America (Ember),,2017,402.87396 -North America (Ember),,2018,399.03775 -North America (Ember),,2019,380.04837 -North America (Ember),,2020,355.66095 -North America (Ember),,2021,366.90955 -North America (Ember),,2022,356.00912 -North America (Ember),,2023,343.5221 +Nigeria,NGA,2014,547.39777 +Nigeria,NGA,2015,529.69556 +Nigeria,NGA,2016,511.22058 +Nigeria,NGA,2017,500 +Nigeria,NGA,2018,519.477 +Nigeria,NGA,2019,505.75977 +Nigeria,NGA,2020,523.421 +Nigeria,NGA,2021,522.70404 +Nigeria,NGA,2022,497.3684 +Nigeria,NGA,2023,508.82144 +Nigeria,NGA,2024,507.84946 +North America,,2000,555.92456 +North America,,2001,560.44336 +North America,,2002,552.51807 +North America,,2003,559.78503 +North America,,2004,553.781 +North America,,2005,553.50867 +North America,,2006,545.7748 +North America,,2007,549.23444 +North America,,2008,539.3964 +North America,,2009,518.7003 +North America,,2010,525.6945 +North America,,2011,506.40802 +North America,,2012,490.81323 +North America,,2013,489.30487 +North America,,2014,485.74512 +North America,,2015,465.74884 +North America,,2016,448.24576 +North America,,2017,434.62997 +North America,,2018,430.4695 +North America,,2019,412.91592 +North America,,2020,385.71655 +North America,,2021,394.33676 +North America,,2022,384.04166 +North America,,2023,373.7501 +North America (Ember),,2000,557.91486 +North America (Ember),,2001,561.6286 +North America (Ember),,2002,552.65924 +North America (Ember),,2003,559.50397 +North America (Ember),,2004,554.1402 +North America (Ember),,2005,553.80884 +North America (Ember),,2006,545.91766 +North America (Ember),,2007,549.24097 +North America (Ember),,2008,540.71576 +North America (Ember),,2009,516.3764 +North America (Ember),,2010,525.0075 +North America (Ember),,2011,503.95447 +North America (Ember),,2012,486.55093 +North America (Ember),,2013,485.1818 +North America (Ember),,2014,482.54752 +North America (Ember),,2015,460.2135 +North America (Ember),,2016,440.80243 +North America (Ember),,2017,426.87436 +North America (Ember),,2018,422.86542 +North America (Ember),,2019,402.65842 +North America (Ember),,2020,376.70297 +North America (Ember),,2021,388.4319 +North America (Ember),,2022,376.55014 +North America (Ember),,2023,363.31644 +North America (Ember),,2024,357.41916 North Korea,PRK,2000,434.71506 North Korea,PRK,2001,433.54904 North Korea,PRK,2002,422.17703 -North Korea,PRK,2003,404.6935 +North Korea,PRK,2003,404.69348 North Korea,PRK,2004,395.6999 -North Korea,PRK,2005,393.93942 -North Korea,PRK,2006,405.64774 +North Korea,PRK,2005,393.9394 +North Korea,PRK,2006,405.6477 North Korea,PRK,2007,355.3062 -North Korea,PRK,2008,365.13446 +North Korea,PRK,2008,365.13443 North Korea,PRK,2009,432.8486 North Korea,PRK,2010,404.3275 North Korea,PRK,2011,346.90942 North Korea,PRK,2012,348.3146 -North Korea,PRK,2013,347.29175 +North Korea,PRK,2013,347.29178 North Korea,PRK,2014,371.04282 North Korea,PRK,2015,433.17484 North Korea,PRK,2016,425.87152 -North Korea,PRK,2017,452.8382 -North Korea,PRK,2018,446.41418 -North Korea,PRK,2019,487.2117 -North Korea,PRK,2020,425.36383 -North Korea,PRK,2021,342.98007 -North Korea,PRK,2022,389.58707 -North Macedonia,MKD,1990,925.2174 -North Macedonia,MKD,1991,866.55115 -North Macedonia,MKD,1992,873.1466 -North Macedonia,MKD,1993,909.2664 -North Macedonia,MKD,1994,898.98987 -North Macedonia,MKD,1995,884.1762 -North Macedonia,MKD,1996,885.5422 -North Macedonia,MKD,1997,879.64343 -North Macedonia,MKD,1998,860.99286 -North Macedonia,MKD,1999,811.9533 +North Korea,PRK,2017,452.83823 +North Korea,PRK,2018,446.2344 +North Korea,PRK,2019,486.59967 +North Korea,PRK,2020,424.4813 +North Korea,PRK,2021,342.5673 +North Korea,PRK,2022,344.25613 North Macedonia,MKD,2000,822.3201 North Macedonia,MKD,2001,907.37836 North Macedonia,MKD,2002,880.13135 -North Macedonia,MKD,2003,811.2928 +North Macedonia,MKD,2003,811.2927 North Macedonia,MKD,2004,794.2943 -North Macedonia,MKD,2005,802.5936 -North Macedonia,MKD,2006,768.9015 -North Macedonia,MKD,2007,835.38464 -North Macedonia,MKD,2008,873.2171 -North Macedonia,MKD,2009,816.7155 +North Macedonia,MKD,2005,802.5937 +North Macedonia,MKD,2006,768.90155 +North Macedonia,MKD,2007,835.3846 +North Macedonia,MKD,2008,873.21716 +North Macedonia,MKD,2009,816.7156 North Macedonia,MKD,2010,677.6859 North Macedonia,MKD,2011,794.37866 North Macedonia,MKD,2012,819.48883 -North Macedonia,MKD,2013,719.2118 -North Macedonia,MKD,2014,749.0706 +North Macedonia,MKD,2013,719.21185 +North Macedonia,MKD,2014,749.0707 North Macedonia,MKD,2015,631.8584 -North Macedonia,MKD,2016,593.25037 -North Macedonia,MKD,2017,705.3571 -North Macedonia,MKD,2018,595.36536 -North Macedonia,MKD,2019,700.17035 +North Macedonia,MKD,2016,593.2504 +North Macedonia,MKD,2017,705.3572 +North Macedonia,MKD,2018,595.3654 +North Macedonia,MKD,2019,700.1704 North Macedonia,MKD,2020,623.59546 -North Macedonia,MKD,2021,564.4283 -North Macedonia,MKD,2022,639.86017 -North Macedonia,MKD,2023,565.3451 -Norway,NOR,1990,25.575657 -Norway,NOR,1991,25.596962 -Norway,NOR,1992,25.446163 -Norway,NOR,1993,25.476109 -Norway,NOR,1994,26.353277 -Norway,NOR,1995,26.175869 -Norway,NOR,1996,27.12771 -Norway,NOR,1997,26.606335 -Norway,NOR,1998,26.370037 -Norway,NOR,1999,26.657944 -Norway,NOR,2000,25.75077 +North Macedonia,MKD,2021,562.38696 +North Macedonia,MKD,2022,624.5791 +North Macedonia,MKD,2023,567.2938 +North Macedonia,MKD,2024,568.9655 +Norway,NOR,2000,25.750772 Norway,NOR,2001,26.871084 Norway,NOR,2002,26.489557 Norway,NOR,2003,27.749134 Norway,NOR,2004,27.883741 -Norway,NOR,2005,26.958107 +Norway,NOR,2005,26.958105 Norway,NOR,2006,27.883188 Norway,NOR,2007,28.579826 Norway,NOR,2008,27.4807 Norway,NOR,2009,39.092922 Norway,NOR,2010,42.878025 -Norway,NOR,2011,38.354645 -Norway,NOR,2012,32.152138 +Norway,NOR,2011,38.35464 +Norway,NOR,2012,32.15214 Norway,NOR,2013,33.949226 -Norway,NOR,2014,33.92959 +Norway,NOR,2014,33.929585 Norway,NOR,2015,33.636364 Norway,NOR,2016,33.1014 Norway,NOR,2017,32.980373 Norway,NOR,2018,33.09579 Norway,NOR,2019,33.475304 -Norway,NOR,2020,30.391392 +Norway,NOR,2020,30.391394 Norway,NOR,2021,26.920383 -Norway,NOR,2022,29.484032 -Norway,NOR,2023,30.080084 -OECD (Ember),,2000,476.5809 -OECD (Ember),,2001,476.87476 -OECD (Ember),,2002,475.99457 -OECD (Ember),,2003,485.66672 -OECD (Ember),,2004,476.55466 -OECD (Ember),,2005,477.10455 -OECD (Ember),,2006,472.42926 -OECD (Ember),,2007,478.7931 -OECD (Ember),,2008,470.3167 -OECD (Ember),,2009,455.4981 -OECD (Ember),,2010,454.098 -OECD (Ember),,2011,451.84918 -OECD (Ember),,2012,453.8718 -OECD (Ember),,2013,449.20035 -OECD (Ember),,2014,442.02365 -OECD (Ember),,2015,428.49854 -OECD (Ember),,2016,416.51743 -OECD (Ember),,2017,408.34213 -OECD (Ember),,2018,400.04004 -OECD (Ember),,2019,378.85016 -OECD (Ember),,2020,357.1209 -OECD (Ember),,2021,362.04123 -OECD (Ember),,2022,360.52798 -OECD (Ember),,2023,341.1077 +Norway,NOR,2022,29.638288 +Norway,NOR,2023,30.482641 +Norway,NOR,2024,30.745583 +OECD (Ember),,2000,491.41174 +OECD (Ember),,2001,491.52084 +OECD (Ember),,2002,490.49753 +OECD (Ember),,2003,500.27722 +OECD (Ember),,2004,490.98862 +OECD (Ember),,2005,491.61902 +OECD (Ember),,2006,486.7227 +OECD (Ember),,2007,493.27298 +OECD (Ember),,2008,484.55994 +OECD (Ember),,2009,469.0507 +OECD (Ember),,2010,467.82828 +OECD (Ember),,2011,465.0388 +OECD (Ember),,2012,466.57037 +OECD (Ember),,2013,461.93127 +OECD (Ember),,2014,454.8367 +OECD (Ember),,2015,440.6886 +OECD (Ember),,2016,428.1426 +OECD (Ember),,2017,420.57822 +OECD (Ember),,2018,412.34113 +OECD (Ember),,2019,390.2965 +OECD (Ember),,2020,367.45242 +OECD (Ember),,2021,372.96722 +OECD (Ember),,2022,371.28616 +OECD (Ember),,2023,349.66846 +OECD (Ember),,2024,337.48355 Oceania,,2000,705.34357 -Oceania,,2001,711.6348 -Oceania,,2002,698.0319 +Oceania,,2001,711.63477 +Oceania,,2002,698.03204 Oceania,,2003,701.5968 Oceania,,2004,695.7793 Oceania,,2005,702.1269 -Oceania,,2006,701.80566 +Oceania,,2006,701.80585 Oceania,,2007,699.22906 Oceania,,2008,703.0783 -Oceania,,2009,692.75854 -Oceania,,2010,681.137 +Oceania,,2009,692.7585 +Oceania,,2010,681.13715 Oceania,,2011,659.4303 -Oceania,,2012,660.32465 +Oceania,,2012,660.3247 Oceania,,2013,641.4103 -Oceania,,2014,643.75446 +Oceania,,2014,643.7544 Oceania,,2015,639.9365 Oceania,,2016,624.76855 -Oceania,,2017,630.5911 -Oceania,,2018,605.9208 -Oceania,,2019,592.6939 -Oceania,,2020,571.31256 -Oceania,,2021,540.431 -Oceania,,2022,507.6928 +Oceania,,2017,630.59106 +Oceania,,2018,606.16815 +Oceania,,2019,592.8471 +Oceania,,2020,571.0893 +Oceania,,2021,541.08673 +Oceania,,2022,514.1319 +Oceania,,2023,493.4306 Oceania (Ember),,2000,705.3056 Oceania (Ember),,2001,711.6715 -Oceania (Ember),,2002,698.1423 +Oceania (Ember),,2002,698.14233 Oceania (Ember),,2003,701.4877 -Oceania (Ember),,2004,695.8505 +Oceania (Ember),,2004,695.8506 Oceania (Ember),,2005,702.0207 -Oceania (Ember),,2006,701.8057 +Oceania (Ember),,2006,701.8058 Oceania (Ember),,2007,699.0938 -Oceania (Ember),,2008,703.01117 +Oceania (Ember),,2008,703.0111 Oceania (Ember),,2009,692.8909 Oceania (Ember),,2010,680.9732 Oceania (Ember),,2011,659.36584 -Oceania (Ember),,2012,660.3903 +Oceania (Ember),,2012,660.3904 Oceania (Ember),,2013,641.4433 Oceania (Ember),,2014,643.68805 Oceania (Ember),,2015,639.9689 Oceania (Ember),,2016,624.8644 Oceania (Ember),,2017,630.6547 -Oceania (Ember),,2018,605.858 -Oceania (Ember),,2019,592.60065 -Oceania (Ember),,2020,571.4063 -Oceania (Ember),,2021,540.431 -Oceania (Ember),,2022,507.6321 -Oceania (Ember),,2023,489.62454 +Oceania (Ember),,2018,606.1367 +Oceania (Ember),,2019,592.785 +Oceania (Ember),,2020,571.21436 +Oceania (Ember),,2021,541.17975 +Oceania (Ember),,2022,514.1623 +Oceania (Ember),,2023,494.75925 +Oceania (Ember),,2024,493.4236 Oman,OMN,2000,580.6806 Oman,OMN,2001,581.1088 Oman,OMN,2002,580.8325 Oman,OMN,2003,581.69934 -Oman,OMN,2004,580.8695 +Oman,OMN,2004,580.86957 Oman,OMN,2005,579.44666 Oman,OMN,2006,578.63055 Oman,OMN,2007,565.9845 -Oman,OMN,2008,566.3552 +Oman,OMN,2008,566.35516 Oman,OMN,2009,566.3956 -Oman,OMN,2010,566.88544 +Oman,OMN,2010,566.8854 Oman,OMN,2011,566.9867 Oman,OMN,2012,566.7466 Oman,OMN,2013,567.0732 Oman,OMN,2014,567.113 -Oman,OMN,2015,567.3282 +Oman,OMN,2015,567.32825 Oman,OMN,2016,567.3386 Oman,OMN,2017,566.80493 -Oman,OMN,2018,566.9228 -Oman,OMN,2019,567.0528 -Oman,OMN,2020,565.12714 -Oman,OMN,2021,564.68024 -Oman,OMN,2022,564.6922 -Oman,OMN,2023,564.63495 +Oman,OMN,2018,566.4725 +Oman,OMN,2019,566.46954 +Oman,OMN,2020,563.6646 +Oman,OMN,2021,557.86707 +Oman,OMN,2022,547.3708 +Oman,OMN,2023,545.8787 +Oman,OMN,2024,545.32874 Pakistan,PAK,2000,499.92484 Pakistan,PAK,2001,483.70547 Pakistan,PAK,2002,465.61963 -Pakistan,PAK,2003,442.37332 +Pakistan,PAK,2003,442.3733 Pakistan,PAK,2004,461.84274 Pakistan,PAK,2005,441.73013 Pakistan,PAK,2006,442.98203 Pakistan,PAK,2007,456.68225 -Pakistan,PAK,2008,459.0904 +Pakistan,PAK,2008,459.09042 Pakistan,PAK,2009,458.22495 Pakistan,PAK,2010,429.6594 Pakistan,PAK,2011,443.3196 Pakistan,PAK,2012,431.70233 Pakistan,PAK,2013,436.21283 Pakistan,PAK,2014,432.76804 -Pakistan,PAK,2015,431.77585 -Pakistan,PAK,2016,456.95685 +Pakistan,PAK,2015,431.77588 +Pakistan,PAK,2016,456.95688 Pakistan,PAK,2017,488.4934 -Pakistan,PAK,2018,487.52133 +Pakistan,PAK,2018,487.5213 Pakistan,PAK,2019,477.98035 -Pakistan,PAK,2020,489.69183 -Pakistan,PAK,2021,480.75095 -Pakistan,PAK,2022,463.6563 -Pakistan,PAK,2023,440.6085 +Pakistan,PAK,2020,485.4315 +Pakistan,PAK,2021,462.90802 +Pakistan,PAK,2022,436.58423 +Pakistan,PAK,2023,425.68472 +Pakistan,PAK,2024,398.6102 Palestine,PSE,2000,666.6667 Palestine,PSE,2001,750 Palestine,PSE,2002,666.6666 -Palestine,PSE,2003,647.0588 -Palestine,PSE,2004,650 +Palestine,PSE,2003,647.05884 +Palestine,PSE,2004,649.99994 Palestine,PSE,2005,660 Palestine,PSE,2006,657.1429 Palestine,PSE,2007,666.6667 Palestine,PSE,2008,651.1628 Palestine,PSE,2009,660 Palestine,PSE,2010,659.57446 -Palestine,PSE,2011,666.6666 +Palestine,PSE,2011,666.6667 Palestine,PSE,2012,652.17395 Palestine,PSE,2013,660.3774 -Palestine,PSE,2014,647.0588 +Palestine,PSE,2014,647.05884 Palestine,PSE,2015,653.8462 Palestine,PSE,2016,603.7736 Palestine,PSE,2017,600 -Palestine,PSE,2018,571.4285 +Palestine,PSE,2018,571.4286 Palestine,PSE,2019,567.90125 Palestine,PSE,2020,523.8095 -Palestine,PSE,2021,516.85394 -Palestine,PSE,2022,516.129 +Palestine,PSE,2021,505.4945 +Palestine,PSE,2022,460.78433 Panama,PAN,2000,210.63394 Panama,PAN,2001,345.70312 Panama,PAN,2002,247.63704 -Panama,PAN,2003,335.72708 +Panama,PAN,2003,335.7271 Panama,PAN,2004,239.58333 Panama,PAN,2005,250.8591 -Panama,PAN,2006,277.59195 +Panama,PAN,2006,277.59198 Panama,PAN,2007,292.56964 Panama,PAN,2008,264.3857 -Panama,PAN,2009,298.11868 -Panama,PAN,2010,296.74796 +Panama,PAN,2009,298.11865 +Panama,PAN,2010,296.748 Panama,PAN,2011,336.31714 Panama,PAN,2012,276.4228 Panama,PAN,2013,307.26257 Panama,PAN,2014,320.4301 Panama,PAN,2015,258.25244 -Panama,PAN,2016,247.24266 -Panama,PAN,2017,205.61595 +Panama,PAN,2016,247.24265 +Panama,PAN,2017,205.61594 Panama,PAN,2018,169.64285 Panama,PAN,2019,325.60135 Panama,PAN,2020,180.98721 Panama,PAN,2021,161.84972 -Panama,PAN,2022,161.67665 +Panama,PAN,2022,168.97958 +Panama,PAN,2023,258.74124 Papua New Guinea,PNG,2000,396.31335 -Papua New Guinea,PNG,2001,447.76123 +Papua New Guinea,PNG,2001,447.7612 Papua New Guinea,PNG,2002,459.93036 -Papua New Guinea,PNG,2003,475.15524 -Papua New Guinea,PNG,2004,492.87753 +Papua New Guinea,PNG,2003,475.15527 +Papua New Guinea,PNG,2004,492.8775 Papua New Guinea,PNG,2005,458.56354 Papua New Guinea,PNG,2006,452.94116 -Papua New Guinea,PNG,2007,429.8508 +Papua New Guinea,PNG,2007,429.85077 Papua New Guinea,PNG,2008,436.95013 Papua New Guinea,PNG,2009,433.23444 Papua New Guinea,PNG,2010,441.59543 Papua New Guinea,PNG,2011,429.75204 Papua New Guinea,PNG,2012,431.63538 -Papua New Guinea,PNG,2013,443.89026 +Papua New Guinea,PNG,2013,443.89023 Papua New Guinea,PNG,2014,453.27103 -Papua New Guinea,PNG,2015,465.7836 -Papua New Guinea,PNG,2016,477.98746 -Papua New Guinea,PNG,2017,491.95404 -Papua New Guinea,PNG,2018,500 -Papua New Guinea,PNG,2019,489.70255 -Papua New Guinea,PNG,2020,484.48685 -Papua New Guinea,PNG,2021,502.1645 -Papua New Guinea,PNG,2022,507.24634 -Paraguay,PRY,2000,23.742756 +Papua New Guinea,PNG,2015,465.78363 +Papua New Guinea,PNG,2016,477.98743 +Papua New Guinea,PNG,2017,491.95407 +Papua New Guinea,PNG,2018,518.34863 +Papua New Guinea,PNG,2019,495.32706 +Papua New Guinea,PNG,2020,490.2439 +Papua New Guinea,PNG,2021,507.72623 +Papua New Guinea,PNG,2022,529.41174 +Papua New Guinea,PNG,2023,513.74207 +Paraguay,PRY,2000,23.742754 Paraguay,PRY,2001,23.83054 Paraguay,PRY,2002,23.853971 Paraguay,PRY,2003,23.758934 -Paraguay,PRY,2004,23.68573 -Paraguay,PRY,2005,23.842098 -Paraguay,PRY,2006,23.805094 +Paraguay,PRY,2004,23.685732 +Paraguay,PRY,2005,23.842096 +Paraguay,PRY,2006,23.805096 Paraguay,PRY,2007,23.831688 -Paraguay,PRY,2008,23.800938 +Paraguay,PRY,2008,23.80094 Paraguay,PRY,2009,23.844193 Paraguay,PRY,2010,23.677395 Paraguay,PRY,2011,23.776466 @@ -4203,19 +4157,19 @@ Paraguay,PRY,2016,24.397873 Paraguay,PRY,2017,24.239386 Paraguay,PRY,2018,24.266937 Paraguay,PRY,2019,24.198427 -Paraguay,PRY,2020,24.295853 -Paraguay,PRY,2021,24.318348 -Paraguay,PRY,2022,24.31266 -Paraguay,PRY,2023,23.75506 -Peru,PER,2000,141.06424 -Peru,PER,2001,117.95859 -Peru,PER,2002,139.15414 -Peru,PER,2003,145.66069 -Peru,PER,2004,200.00002 +Paraguay,PRY,2020,24.876688 +Paraguay,PRY,2021,24.975513 +Paraguay,PRY,2022,24.71095 +Paraguay,PRY,2023,24.864376 +Peru,PER,2000,141.06425 +Peru,PER,2001,117.958595 +Peru,PER,2002,139.15416 +Peru,PER,2003,145.6607 +Peru,PER,2004,200 Peru,PER,2005,200 -Peru,PER,2006,192.98247 +Peru,PER,2006,192.98245 Peru,PER,2007,229.53558 -Peru,PER,2008,269.1952 +Peru,PER,2008,269.19516 Peru,PER,2009,265.18832 Peru,PER,2010,283.08722 Peru,PER,2011,286.5154 @@ -4223,112 +4177,96 @@ Peru,PER,2012,297.54758 Peru,PER,2013,304.9874 Peru,PER,2014,309.64355 Peru,PER,2015,305.78 -Peru,PER,2016,320.9374 +Peru,PER,2016,320.93744 Peru,PER,2017,271.4855 -Peru,PER,2018,255.14102 -Peru,PER,2019,257.80426 -Peru,PER,2020,237.82452 -Peru,PER,2021,249.42752 -Peru,PER,2022,251.73491 -Peru,PER,2023,266.47754 +Peru,PER,2018,252.8378 +Peru,PER,2019,256.7259 +Peru,PER,2020,239.66785 +Peru,PER,2021,250.9212 +Peru,PER,2022,287.81302 +Peru,PER,2023,300.53064 +Peru,PER,2024,263.2733 Philippines,PHL,2000,467.99118 Philippines,PHL,2001,505.84485 Philippines,PHL,2002,466.98306 Philippines,PHL,2003,455.42126 Philippines,PHL,2004,460.4934 Philippines,PHL,2005,448.47095 -Philippines,PHL,2006,427.26315 +Philippines,PHL,2006,427.26312 Philippines,PHL,2007,452.02954 -Philippines,PHL,2008,430.94373 +Philippines,PHL,2008,430.94376 Philippines,PHL,2009,441.21448 -Philippines,PHL,2010,504.50314 +Philippines,PHL,2010,504.5031 Philippines,PHL,2011,492.84167 Philippines,PHL,2012,505.00757 Philippines,PHL,2013,529.8418 Philippines,PHL,2014,536.7085 Philippines,PHL,2015,544.6602 Philippines,PHL,2016,560.0837 -Philippines,PHL,2017,560.5129 +Philippines,PHL,2017,560.51294 Philippines,PHL,2018,572.38824 -Philippines,PHL,2019,592.6624 -Philippines,PHL,2020,598.8207 -Philippines,PHL,2021,597.1541 -Philippines,PHL,2022,601.0972 -Philippines,PHL,2023,610.68835 -Poland,POL,1990,967.04614 -Poland,POL,1991,967.6762 -Poland,POL,1992,966.71265 -Poland,POL,1993,967.2181 -Poland,POL,1994,963.7633 -Poland,POL,1995,961.83875 -Poland,POL,1996,960.6912 -Poland,POL,1997,959.5573 -Poland,POL,1998,957.3062 -Poland,POL,1999,956.78253 -Poland,POL,2000,954.60266 +Philippines,PHL,2019,592.6625 +Philippines,PHL,2020,598.8206 +Philippines,PHL,2021,599.62177 +Philippines,PHL,2022,602.6722 +Philippines,PHL,2023,612.5385 +Philippines,PHL,2024,613.37994 +Poland,POL,2000,954.6026 Poland,POL,2001,949.5547 -Poland,POL,2002,945.61017 +Poland,POL,2002,945.6102 Poland,POL,2003,949.80334 -Poland,POL,2004,941.2614 +Poland,POL,2004,941.26135 Poland,POL,2005,932.25586 Poland,POL,2006,931.92285 -Poland,POL,2007,925.4984 -Poland,POL,2008,918.576 -Poland,POL,2009,904.8786 -Poland,POL,2010,893.0917 -Poland,POL,2011,881.74615 +Poland,POL,2007,925.49835 +Poland,POL,2008,918.57605 +Poland,POL,2009,904.87866 +Poland,POL,2010,893.09174 +Poland,POL,2011,881.7462 Poland,POL,2012,860.7877 Poland,POL,2013,863.2312 Poland,POL,2014,844.0796 -Poland,POL,2015,826.1717 +Poland,POL,2015,826.17163 Poland,POL,2016,818.61444 -Poland,POL,2017,805.7795 +Poland,POL,2017,805.77954 Poland,POL,2018,809.30176 Poland,POL,2019,771.90295 -Poland,POL,2020,738.721 +Poland,POL,2020,738.72095 Poland,POL,2021,756.86566 -Poland,POL,2022,733.18396 -Poland,POL,2023,661.92584 -Portugal,PRT,1990,539.7105 -Portugal,PRT,1991,552.8264 -Portugal,PRT,1992,650.27136 -Portugal,PRT,1993,581.38776 -Portugal,PRT,1994,539.2031 -Portugal,PRT,1995,606.50256 -Portugal,PRT,1996,482.16373 -Portugal,PRT,1997,512.83563 -Portugal,PRT,1998,515.35486 -Portugal,PRT,1999,606.4576 -Portugal,PRT,2000,546.1699 +Poland,POL,2022,732.9787 +Poland,POL,2023,655.2752 +Poland,POL,2024,614.98236 +Portugal,PRT,2000,546.16986 Portugal,PRT,2001,506.94745 Portugal,PRT,2002,592.5357 Portugal,PRT,2003,485.2339 Portugal,PRT,2004,541.18176 Portugal,PRT,2005,595.30536 -Portugal,PRT,2006,503.92557 -Portugal,PRT,2007,472.27573 +Portugal,PRT,2006,503.9256 +Portugal,PRT,2007,472.27576 Portugal,PRT,2008,471.54828 Portugal,PRT,2009,452.8149 Portugal,PRT,2010,317.12787 -Portugal,PRT,2011,375.0483 -Portugal,PRT,2012,441.2412 +Portugal,PRT,2011,375.04828 +Portugal,PRT,2012,441.24118 Portugal,PRT,2013,340.41708 Portugal,PRT,2014,325.6757 -Portugal,PRT,2015,413.9617 -Portugal,PRT,2016,343.60712 -Portugal,PRT,2017,433.75583 +Portugal,PRT,2015,413.96167 +Portugal,PRT,2016,343.60715 +Portugal,PRT,2017,433.75586 Portugal,PRT,2018,358.69754 Portugal,PRT,2019,300.969 -Portugal,PRT,2020,248.29732 +Portugal,PRT,2020,248.29735 Portugal,PRT,2021,210.48354 -Portugal,PRT,2022,225.1941 -Portugal,PRT,2023,165.55257 -Puerto Rico,PRI,2000,657.20825 +Portugal,PRT,2022,225.33994 +Portugal,PRT,2023,158.25937 +Portugal,PRT,2024,112.28676 +Puerto Rico,PRI,2000,657.2082 Puerto Rico,PRI,2001,655.75977 Puerto Rico,PRI,2002,664.076 -Puerto Rico,PRI,2003,684.78253 +Puerto Rico,PRI,2003,684.7826 Puerto Rico,PRI,2004,685.3002 -Puerto Rico,PRI,2005,684.29486 +Puerto Rico,PRI,2005,684.2949 Puerto Rico,PRI,2006,689.62756 Puerto Rico,PRI,2007,689.22205 Puerto Rico,PRI,2008,690.3226 @@ -4337,51 +4275,54 @@ Puerto Rico,PRI,2010,686.4516 Puerto Rico,PRI,2011,691.82385 Puerto Rico,PRI,2012,687.8924 Puerto Rico,PRI,2013,684.8231 -Puerto Rico,PRI,2014,686.60974 +Puerto Rico,PRI,2014,686.6097 Puerto Rico,PRI,2015,680.8612 -Puerto Rico,PRI,2016,683.1395 +Puerto Rico,PRI,2016,683.1396 Puerto Rico,PRI,2017,673.9927 -Puerto Rico,PRI,2018,679.83966 -Puerto Rico,PRI,2019,682.37164 -Puerto Rico,PRI,2020,678.7427 -Puerto Rico,PRI,2021,678.6279 -Puerto Rico,PRI,2022,678.7377 -Qatar,QAT,2000,603.5049 +Puerto Rico,PRI,2018,671.56586 +Puerto Rico,PRI,2019,673.6292 +Puerto Rico,PRI,2020,669.11383 +Puerto Rico,PRI,2021,665.28925 +Puerto Rico,PRI,2022,660.804 +Puerto Rico,PRI,2023,660.804 +Puerto Rico,PRI,2024,664.5299 +Qatar,QAT,2000,603.50494 Qatar,QAT,2001,604.02014 Qatar,QAT,2002,603.2907 -Qatar,QAT,2003,603.66364 +Qatar,QAT,2003,603.6636 Qatar,QAT,2004,603.9305 Qatar,QAT,2005,603.4722 -Qatar,QAT,2006,603.63007 +Qatar,QAT,2006,603.63 Qatar,QAT,2007,603.8027 Qatar,QAT,2008,603.6077 Qatar,QAT,2009,603.4768 -Qatar,QAT,2010,603.76685 -Qatar,QAT,2011,603.6446 -Qatar,QAT,2012,603.44824 -Qatar,QAT,2013,602.8777 -Qatar,QAT,2014,602.5773 -Qatar,QAT,2015,602.4502 -Qatar,QAT,2016,602.4976 +Qatar,QAT,2010,603.7669 +Qatar,QAT,2011,603.64465 +Qatar,QAT,2012,603.4483 +Qatar,QAT,2013,602.87775 +Qatar,QAT,2014,602.57733 +Qatar,QAT,2015,602.45013 +Qatar,QAT,2016,602.4977 Qatar,QAT,2017,602.407 Qatar,QAT,2018,602.372 Qatar,QAT,2019,602.8 Qatar,QAT,2020,602.6316 Qatar,QAT,2021,602.4329 -Qatar,QAT,2022,602.59015 -Qatar,QAT,2023,602.5005 +Qatar,QAT,2022,602.7322 +Qatar,QAT,2023,602.6478 +Qatar,QAT,2024,602.8308 Reunion,REU,2000,482.95456 Reunion,REU,2001,518.9189 -Reunion,REU,2002,492.22797 +Reunion,REU,2002,492.228 Reunion,REU,2003,500 -Reunion,REU,2004,493.15067 -Reunion,REU,2005,568.282 -Reunion,REU,2006,559.322 -Reunion,REU,2007,577.2358 +Reunion,REU,2004,493.1507 +Reunion,REU,2005,568.2819 +Reunion,REU,2006,559.3221 +Reunion,REU,2007,577.2357 Reunion,REU,2008,578.7402 -Reunion,REU,2009,595.4198 -Reunion,REU,2010,585.18524 -Reunion,REU,2011,610.909 +Reunion,REU,2009,595.41986 +Reunion,REU,2010,585.1852 +Reunion,REU,2011,610.90906 Reunion,REU,2012,581.5603 Reunion,REU,2013,553.19147 Reunion,REU,2014,576.9231 @@ -4390,21 +4331,12 @@ Reunion,REU,2016,561.2245 Reunion,REU,2017,570.4698 Reunion,REU,2018,535.5932 Reunion,REU,2019,567.21313 -Reunion,REU,2020,643.79956 +Reunion,REU,2020,643.7995 Reunion,REU,2021,572.81555 -Romania,ROU,1990,584.29236 -Romania,ROU,1991,530.63385 -Romania,ROU,1992,588.2244 -Romania,ROU,1993,588.6922 -Romania,ROU,1994,592.1148 -Romania,ROU,1995,564.7756 -Romania,ROU,1996,560.2163 -Romania,ROU,1997,480.07755 -Romania,ROU,1998,436.30872 -Romania,ROU,1999,441.25223 +Reunion,REU,2022,525.22253 Romania,ROU,2000,513.7703 Romania,ROU,2001,524.5565 -Romania,ROU,2002,515.06805 +Romania,ROU,2002,515.068 Romania,ROU,2003,573.8512 Romania,ROU,2004,518.15125 Romania,ROU,2005,489.9713 @@ -4414,25 +4346,26 @@ Romania,ROU,2008,497.53314 Romania,ROU,2009,469.6996 Romania,ROU,2010,424.52832 Romania,ROU,2011,490.6149 -Romania,ROU,2012,486.84662 +Romania,ROU,2012,486.8466 Romania,ROU,2013,389.7577 Romania,ROU,2014,357.05484 Romania,ROU,2015,369.02222 Romania,ROU,2016,345.37527 -Romania,ROU,2017,369.2162 +Romania,ROU,2017,369.21622 Romania,ROU,2018,348.0568 -Romania,ROU,2019,328.96527 -Romania,ROU,2020,277.8482 +Romania,ROU,2019,328.9653 +Romania,ROU,2020,277.84824 Romania,ROU,2021,289.959 -Romania,ROU,2022,297.93457 -Romania,ROU,2023,240.58281 +Romania,ROU,2022,297.77127 +Romania,ROU,2023,243.19579 +Romania,ROU,2024,245.55428 Russia,RUS,2000,470.04874 -Russia,RUS,2001,458.40207 +Russia,RUS,2001,458.40204 Russia,RUS,2002,463.14288 -Russia,RUS,2003,465.37674 +Russia,RUS,2003,465.3767 Russia,RUS,2004,453.82068 -Russia,RUS,2005,457.49445 -Russia,RUS,2006,462.59525 +Russia,RUS,2005,457.49442 +Russia,RUS,2006,462.59528 Russia,RUS,2007,458.28015 Russia,RUS,2008,475.21155 Russia,RUS,2009,452.18005 @@ -4440,26 +4373,27 @@ Russia,RUS,2010,458.1029 Russia,RUS,2011,461.3977 Russia,RUS,2012,461.9039 Russia,RUS,2013,450.17374 -Russia,RUS,2014,449.28586 -Russia,RUS,2015,444.35385 -Russia,RUS,2016,442.7436 +Russia,RUS,2014,449.2859 +Russia,RUS,2015,444.35388 +Russia,RUS,2016,442.74362 Russia,RUS,2017,439.629 -Russia,RUS,2018,439.3296 +Russia,RUS,2018,439.32962 Russia,RUS,2019,440.0354 Russia,RUS,2020,417.65396 -Russia,RUS,2021,424.7885 -Russia,RUS,2022,436.2778 -Russia,RUS,2023,441.03885 +Russia,RUS,2021,426.82812 +Russia,RUS,2022,438.42017 +Russia,RUS,2023,445.02438 +Russia,RUS,2024,449.19592 Rwanda,RWA,2000,250 Rwanda,RWA,2001,266.66666 Rwanda,RWA,2002,294.11765 -Rwanda,RWA,2003,222.22223 -Rwanda,RWA,2004,285.7143 +Rwanda,RWA,2003,222.22221 +Rwanda,RWA,2004,285.71426 Rwanda,RWA,2005,315.78946 -Rwanda,RWA,2006,333.33337 -Rwanda,RWA,2007,360.00003 +Rwanda,RWA,2006,333.33334 +Rwanda,RWA,2007,360 Rwanda,RWA,2008,310.34485 -Rwanda,RWA,2009,322.58063 +Rwanda,RWA,2009,322.58066 Rwanda,RWA,2010,428.5714 Rwanda,RWA,2011,400 Rwanda,RWA,2012,358.97437 @@ -4468,11 +4402,12 @@ Rwanda,RWA,2014,341.4634 Rwanda,RWA,2015,294.11768 Rwanda,RWA,2016,343.75 Rwanda,RWA,2017,353.84616 -Rwanda,RWA,2018,395.06174 -Rwanda,RWA,2019,348.83722 -Rwanda,RWA,2020,250 -Rwanda,RWA,2021,285.7143 -Rwanda,RWA,2022,316.32654 +Rwanda,RWA,2018,390.2439 +Rwanda,RWA,2019,329.41174 +Rwanda,RWA,2020,247.19101 +Rwanda,RWA,2021,280 +Rwanda,RWA,2022,288.46155 +Rwanda,RWA,2023,301.8868 Saint Helena,SHN,2000,1000 Saint Helena,SHN,2001,1000 Saint Helena,SHN,2002,1000 @@ -4504,7 +4439,7 @@ Saint Kitts and Nevis,KNA,2004,666.6666 Saint Kitts and Nevis,KNA,2005,684.2105 Saint Kitts and Nevis,KNA,2006,684.2105 Saint Kitts and Nevis,KNA,2007,666.6667 -Saint Kitts and Nevis,KNA,2008,650 +Saint Kitts and Nevis,KNA,2008,649.99994 Saint Kitts and Nevis,KNA,2009,636.36365 Saint Kitts and Nevis,KNA,2010,666.6667 Saint Kitts and Nevis,KNA,2011,636.36365 @@ -4512,36 +4447,36 @@ Saint Kitts and Nevis,KNA,2012,636.36365 Saint Kitts and Nevis,KNA,2013,636.36365 Saint Kitts and Nevis,KNA,2014,636.36365 Saint Kitts and Nevis,KNA,2015,636.36365 -Saint Kitts and Nevis,KNA,2016,608.6957 +Saint Kitts and Nevis,KNA,2016,608.6956 Saint Kitts and Nevis,KNA,2017,636.36365 -Saint Kitts and Nevis,KNA,2018,608.6957 -Saint Kitts and Nevis,KNA,2019,608.6957 +Saint Kitts and Nevis,KNA,2018,608.6956 +Saint Kitts and Nevis,KNA,2019,608.6956 Saint Kitts and Nevis,KNA,2020,636.36365 Saint Kitts and Nevis,KNA,2021,636.36365 Saint Kitts and Nevis,KNA,2022,636.36365 Saint Lucia,LCA,2000,642.8572 -Saint Lucia,LCA,2001,655.17236 -Saint Lucia,LCA,2002,655.17236 +Saint Lucia,LCA,2001,655.1724 +Saint Lucia,LCA,2002,655.1724 Saint Lucia,LCA,2003,666.6666 -Saint Lucia,LCA,2004,645.16125 -Saint Lucia,LCA,2005,656.24994 +Saint Lucia,LCA,2004,645.1613 +Saint Lucia,LCA,2005,656.25 Saint Lucia,LCA,2006,666.6666 Saint Lucia,LCA,2007,657.1429 Saint Lucia,LCA,2008,657.1429 Saint Lucia,LCA,2009,666.6666 -Saint Lucia,LCA,2010,657.8947 -Saint Lucia,LCA,2011,666.6666 -Saint Lucia,LCA,2012,657.8947 -Saint Lucia,LCA,2013,657.8947 -Saint Lucia,LCA,2014,657.8947 -Saint Lucia,LCA,2015,657.8947 -Saint Lucia,LCA,2016,666.6666 -Saint Lucia,LCA,2017,650 -Saint Lucia,LCA,2018,650 -Saint Lucia,LCA,2019,658.5366 -Saint Lucia,LCA,2020,648.6486 -Saint Lucia,LCA,2021,666.6666 -Saint Lucia,LCA,2022,666.6666 +Saint Lucia,LCA,2010,657.8948 +Saint Lucia,LCA,2011,666.6667 +Saint Lucia,LCA,2012,657.8948 +Saint Lucia,LCA,2013,657.8948 +Saint Lucia,LCA,2014,657.8948 +Saint Lucia,LCA,2015,657.8948 +Saint Lucia,LCA,2016,666.6667 +Saint Lucia,LCA,2017,649.99994 +Saint Lucia,LCA,2018,634.1463 +Saint Lucia,LCA,2019,642.8572 +Saint Lucia,LCA,2020,631.5789 +Saint Lucia,LCA,2021,649.99994 +Saint Lucia,LCA,2022,649.99994 Saint Pierre and Miquelon,SPM,2000,750 Saint Pierre and Miquelon,SPM,2001,750 Saint Pierre and Miquelon,SPM,2002,750 @@ -4569,48 +4504,50 @@ Saint Vincent and the Grenadines,VCT,2000,500 Saint Vincent and the Grenadines,VCT,2001,500 Saint Vincent and the Grenadines,VCT,2002,500 Saint Vincent and the Grenadines,VCT,2003,545.4545 -Saint Vincent and the Grenadines,VCT,2004,499.99997 +Saint Vincent and the Grenadines,VCT,2004,500 Saint Vincent and the Grenadines,VCT,2005,500 -Saint Vincent and the Grenadines,VCT,2006,538.4616 -Saint Vincent and the Grenadines,VCT,2007,571.4286 -Saint Vincent and the Grenadines,VCT,2008,571.4286 -Saint Vincent and the Grenadines,VCT,2009,571.4286 +Saint Vincent and the Grenadines,VCT,2006,538.46155 +Saint Vincent and the Grenadines,VCT,2007,571.4285 +Saint Vincent and the Grenadines,VCT,2008,571.4285 +Saint Vincent and the Grenadines,VCT,2009,571.4285 Saint Vincent and the Grenadines,VCT,2010,533.3333 Saint Vincent and the Grenadines,VCT,2011,500 Saint Vincent and the Grenadines,VCT,2012,533.3333 -Saint Vincent and the Grenadines,VCT,2013,571.4286 +Saint Vincent and the Grenadines,VCT,2013,571.4285 Saint Vincent and the Grenadines,VCT,2014,642.8572 -Saint Vincent and the Grenadines,VCT,2015,600.00006 +Saint Vincent and the Grenadines,VCT,2015,600 Saint Vincent and the Grenadines,VCT,2016,562.50006 -Saint Vincent and the Grenadines,VCT,2017,600.00006 -Saint Vincent and the Grenadines,VCT,2018,600.00006 -Saint Vincent and the Grenadines,VCT,2019,600.00006 -Saint Vincent and the Grenadines,VCT,2020,529.4118 -Saint Vincent and the Grenadines,VCT,2021,529.4118 -Saint Vincent and the Grenadines,VCT,2022,529.4118 +Saint Vincent and the Grenadines,VCT,2017,600 +Saint Vincent and the Grenadines,VCT,2018,600 +Saint Vincent and the Grenadines,VCT,2019,600 +Saint Vincent and the Grenadines,VCT,2020,600 +Saint Vincent and the Grenadines,VCT,2021,571.4285 +Saint Vincent and the Grenadines,VCT,2022,600 +Saint Vincent and the Grenadines,VCT,2023,600 Samoa,WSM,2000,375 Samoa,WSM,2001,300 Samoa,WSM,2002,272.72726 Samoa,WSM,2003,300 Samoa,WSM,2004,363.63635 Samoa,WSM,2005,363.63635 -Samoa,WSM,2006,416.66666 -Samoa,WSM,2007,416.66666 +Samoa,WSM,2006,416.6667 +Samoa,WSM,2007,416.6667 Samoa,WSM,2008,363.63635 -Samoa,WSM,2009,384.6154 -Samoa,WSM,2010,416.66666 -Samoa,WSM,2011,384.6154 -Samoa,WSM,2012,454.54544 -Samoa,WSM,2013,499.99997 -Samoa,WSM,2014,538.4616 +Samoa,WSM,2009,384.61542 +Samoa,WSM,2010,416.6667 +Samoa,WSM,2011,384.61542 +Samoa,WSM,2012,454.54547 +Samoa,WSM,2013,500 +Samoa,WSM,2014,538.46155 Samoa,WSM,2015,461.53848 -Samoa,WSM,2016,466.6667 +Samoa,WSM,2016,466.66666 Samoa,WSM,2017,437.5 -Samoa,WSM,2018,400 -Samoa,WSM,2019,437.5 -Samoa,WSM,2020,411.7647 -Samoa,WSM,2021,400 -Samoa,WSM,2022,473.68423 +Samoa,WSM,2018,399.99997 +Samoa,WSM,2019,411.7647 +Samoa,WSM,2020,388.8889 +Samoa,WSM,2021,352.94116 +Samoa,WSM,2022,375 +Samoa,WSM,2023,399.99997 Sao Tome and Principe,STP,2000,333.33334 Sao Tome and Principe,STP,2001,500 Sao Tome and Principe,STP,2002,500 @@ -4620,11 +4557,11 @@ Sao Tome and Principe,STP,2005,750 Sao Tome and Principe,STP,2006,600 Sao Tome and Principe,STP,2007,600 Sao Tome and Principe,STP,2008,600 -Sao Tome and Principe,STP,2009,499.99997 +Sao Tome and Principe,STP,2009,500 Sao Tome and Principe,STP,2010,600 -Sao Tome and Principe,STP,2011,571.4286 +Sao Tome and Principe,STP,2011,571.4285 Sao Tome and Principe,STP,2012,625 -Sao Tome and Principe,STP,2013,571.4286 +Sao Tome and Principe,STP,2013,571.4285 Sao Tome and Principe,STP,2014,625 Sao Tome and Principe,STP,2015,600 Sao Tome and Principe,STP,2016,636.36365 @@ -4633,21 +4570,21 @@ Sao Tome and Principe,STP,2018,700 Sao Tome and Principe,STP,2019,636.36365 Sao Tome and Principe,STP,2020,636.36365 Sao Tome and Principe,STP,2021,636.36365 -Sao Tome and Principe,STP,2022,642.8572 +Sao Tome and Principe,STP,2022,555.55554 Saudi Arabia,SAU,2000,692.3354 Saudi Arabia,SAU,2001,697.173 Saudi Arabia,SAU,2002,699.6966 Saudi Arabia,SAU,2003,698.8834 -Saudi Arabia,SAU,2004,700.4786 +Saudi Arabia,SAU,2004,700.47864 Saudi Arabia,SAU,2005,700.1308 Saudi Arabia,SAU,2006,697.7739 Saudi Arabia,SAU,2007,695.49475 Saudi Arabia,SAU,2008,694.809 -Saudi Arabia,SAU,2009,690.3041 +Saudi Arabia,SAU,2009,690.30414 Saudi Arabia,SAU,2010,691.4017 Saudi Arabia,SAU,2011,691.2599 Saudi Arabia,SAU,2012,692.7529 -Saudi Arabia,SAU,2013,693.0645 +Saudi Arabia,SAU,2013,693.0646 Saudi Arabia,SAU,2014,692.5655 Saudi Arabia,SAU,2015,693.0755 Saudi Arabia,SAU,2016,697.2264 @@ -4655,203 +4592,181 @@ Saudi Arabia,SAU,2017,699.96716 Saudi Arabia,SAU,2018,702.8845 Saudi Arabia,SAU,2019,701.7039 Saudi Arabia,SAU,2020,703.1504 -Saudi Arabia,SAU,2021,701.1757 -Saudi Arabia,SAU,2022,706.7905 +Saudi Arabia,SAU,2021,701.5066 +Saudi Arabia,SAU,2022,700.11957 +Saudi Arabia,SAU,2023,696.31335 Senegal,SEN,2000,644.2953 -Senegal,SEN,2001,642.45807 +Senegal,SEN,2001,642.4581 Senegal,SEN,2002,586.5385 Senegal,SEN,2003,566.5236 -Senegal,SEN,2004,585.774 -Senegal,SEN,2005,596.22644 -Senegal,SEN,2006,602.27277 +Senegal,SEN,2004,585.77405 +Senegal,SEN,2005,596.2264 +Senegal,SEN,2006,602.2727 Senegal,SEN,2007,614.2857 -Senegal,SEN,2008,611.4865 -Senegal,SEN,2009,606.55743 +Senegal,SEN,2008,611.48645 +Senegal,SEN,2009,606.5574 Senegal,SEN,2010,609.9071 Senegal,SEN,2011,607.9028 Senegal,SEN,2012,616.0221 Senegal,SEN,2013,611.5485 Senegal,SEN,2014,617.2249 -Senegal,SEN,2015,619.3693 -Senegal,SEN,2016,623.4309 +Senegal,SEN,2015,619.3694 +Senegal,SEN,2016,623.43097 Senegal,SEN,2017,608.05084 -Senegal,SEN,2018,595.14166 -Senegal,SEN,2019,609.9656 +Senegal,SEN,2018,595.1417 +Senegal,SEN,2019,609.96564 Senegal,SEN,2020,562.7063 Senegal,SEN,2021,546.5466 -Senegal,SEN,2022,511.59793 -Serbia,SRB,1990,772.38434 -Serbia,SRB,1991,712.31085 -Serbia,SRB,1992,719.8399 -Serbia,SRB,1993,719.5836 -Serbia,SRB,1994,705.8823 -Serbia,SRB,1995,685.1852 -Serbia,SRB,1996,634.9062 -Serbia,SRB,1997,691.27893 -Serbia,SRB,1998,697.47437 -Serbia,SRB,1999,629.98096 -Serbia,SRB,2000,688.94366 -Serbia,SRB,2001,686.7868 -Serbia,SRB,2002,696.97876 -Serbia,SRB,2003,742.20557 -Serbia,SRB,2004,710.5333 -Serbia,SRB,2005,675.7127 -Serbia,SRB,2006,711.7018 -Serbia,SRB,2007,731.3269 -Serbia,SRB,2008,738.82794 -Serbia,SRB,2009,723.0488 -Serbia,SRB,2010,681.6274 -Serbia,SRB,2011,769.11115 -Serbia,SRB,2012,739.74457 -Serbia,SRB,2013,740.2207 -Serbia,SRB,2014,671.7557 -Serbia,SRB,2015,731.26135 -Serbia,SRB,2016,716.5735 -Serbia,SRB,2017,740.82074 -Serbia,SRB,2018,696.1518 -Serbia,SRB,2019,707.7128 -Serbia,SRB,2020,721.7391 -Serbia,SRB,2021,653.1519 -Serbia,SRB,2022,695.6884 -Serbia,SRB,2023,636.06226 -Seychelles,SYC,2000,650 -Seychelles,SYC,2001,650 +Senegal,SEN,2022,526.6667 +Senegal,SEN,2023,535.4037 +Serbia,SRB,2000,699.5305 +Serbia,SRB,2001,701.56586 +Serbia,SRB,2002,707.5812 +Serbia,SRB,2003,750.146 +Serbia,SRB,2004,719.8172 +Serbia,SRB,2005,686.83765 +Serbia,SRB,2006,723.72455 +Serbia,SRB,2007,742.5632 +Serbia,SRB,2008,749.59265 +Serbia,SRB,2009,734.288 +Serbia,SRB,2010,693.4795 +Serbia,SRB,2011,780.3787 +Serbia,SRB,2012,751.8652 +Serbia,SRB,2013,751.9755 +Serbia,SRB,2014,683.6125 +Serbia,SRB,2015,744.34686 +Serbia,SRB,2016,729.86273 +Serbia,SRB,2017,752.74426 +Serbia,SRB,2018,709.8446 +Serbia,SRB,2019,721.3782 +Serbia,SRB,2020,736.1485 +Serbia,SRB,2021,666.3107 +Serbia,SRB,2022,710.67737 +Serbia,SRB,2023,641.56244 +Serbia,SRB,2024,670.7974 +Seychelles,SYC,2000,649.99994 +Seychelles,SYC,2001,649.99994 Seychelles,SYC,2002,652.17395 Seychelles,SYC,2003,652.17395 Seychelles,SYC,2004,652.17395 Seychelles,SYC,2005,666.6666 -Seychelles,SYC,2006,656.24994 -Seychelles,SYC,2007,647.0588 -Seychelles,SYC,2008,647.0588 +Seychelles,SYC,2006,656.25 +Seychelles,SYC,2007,647.05884 +Seychelles,SYC,2008,647.05884 Seychelles,SYC,2009,657.1429 -Seychelles,SYC,2010,657.8947 -Seychelles,SYC,2011,650 +Seychelles,SYC,2010,657.8948 +Seychelles,SYC,2011,649.99994 Seychelles,SYC,2012,658.5366 Seychelles,SYC,2013,636.36365 -Seychelles,SYC,2014,644.4444 +Seychelles,SYC,2014,644.44446 Seychelles,SYC,2015,652.17395 Seychelles,SYC,2016,640 Seychelles,SYC,2017,647.05884 Seychelles,SYC,2018,653.8462 Seychelles,SYC,2019,614.0351 Seychelles,SYC,2020,576.27124 -Seychelles,SYC,2021,566.6667 +Seychelles,SYC,2021,566.6666 Seychelles,SYC,2022,564.5161 +Seychelles,SYC,2023,571.4286 Sierra Leone,SLE,2000,555.55554 Sierra Leone,SLE,2001,615.38464 Sierra Leone,SLE,2002,562.50006 -Sierra Leone,SLE,2003,600.00006 -Sierra Leone,SLE,2004,583.3333 +Sierra Leone,SLE,2003,600 +Sierra Leone,SLE,2004,583.3334 Sierra Leone,SLE,2005,500 Sierra Leone,SLE,2006,600 -Sierra Leone,SLE,2007,499.99997 -Sierra Leone,SLE,2008,499.99997 -Sierra Leone,SLE,2009,499.99997 -Sierra Leone,SLE,2010,71.42857 +Sierra Leone,SLE,2007,500 +Sierra Leone,SLE,2008,500 +Sierra Leone,SLE,2009,500 +Sierra Leone,SLE,2010,71.428566 Sierra Leone,SLE,2011,58.82353 -Sierra Leone,SLE,2012,222.22223 +Sierra Leone,SLE,2012,222.22221 Sierra Leone,SLE,2013,250 Sierra Leone,SLE,2014,133.33333 Sierra Leone,SLE,2015,187.5 -Sierra Leone,SLE,2016,238.09523 +Sierra Leone,SLE,2016,238.09525 Sierra Leone,SLE,2017,260.86957 Sierra Leone,SLE,2018,222.22221 Sierra Leone,SLE,2019,50 Sierra Leone,SLE,2020,50 Sierra Leone,SLE,2021,50 Sierra Leone,SLE,2022,50 -Singapore,SGP,2000,617.93494 -Singapore,SGP,2001,562.6134 -Singapore,SGP,2002,551.0675 -Singapore,SGP,2003,536.2401 -Singapore,SGP,2004,522.5421 -Singapore,SGP,2005,512.8205 -Singapore,SGP,2006,507.4702 -Singapore,SGP,2007,505.22733 -Singapore,SGP,2008,502.03986 -Singapore,SGP,2009,501.67465 -Singapore,SGP,2010,507.4956 -Singapore,SGP,2011,505.86957 -Singapore,SGP,2012,493.82187 -Singapore,SGP,2013,476.96475 -Singapore,SGP,2014,476.47058 -Singapore,SGP,2015,476.73032 -Singapore,SGP,2016,476.2551 -Singapore,SGP,2017,476.66412 -Singapore,SGP,2018,476.6849 -Singapore,SGP,2019,475.80347 -Singapore,SGP,2020,474.46768 -Singapore,SGP,2021,475.8917 -Singapore,SGP,2022,473.9986 -Singapore,SGP,2023,470.7832 -Slovakia,SVK,1990,386.42606 -Slovakia,SVK,1991,369.74445 -Slovakia,SVK,1992,390.2754 -Slovakia,SVK,1993,324.32434 -Slovakia,SVK,1994,272.0848 -Slovakia,SVK,1995,321.22586 -Slovakia,SVK,1996,322.4666 -Slovakia,SVK,1997,331.07028 -Slovakia,SVK,1998,320.6374 -Slovakia,SVK,1999,305.5852 -Slovakia,SVK,2000,250.48734 +Sierra Leone,SLE,2023,47.61905 +Singapore,SGP,2000,657.0887 +Singapore,SGP,2001,598.3061 +Singapore,SGP,2002,586.2666 +Singapore,SGP,2003,570.4983 +Singapore,SGP,2004,555.6763 +Singapore,SGP,2005,545.5259 +Singapore,SGP,2006,539.377 +Singapore,SGP,2007,537.80695 +Singapore,SGP,2008,534.19727 +Singapore,SGP,2009,533.4928 +Singapore,SGP,2010,539.6825 +Singapore,SGP,2011,538.04346 +Singapore,SGP,2012,525.5646 +Singapore,SGP,2013,507.60892 +Singapore,SGP,2014,506.89655 +Singapore,SGP,2015,506.96103 +Singapore,SGP,2016,506.4935 +Singapore,SGP,2017,507.2686 +Singapore,SGP,2018,507.07947 +Singapore,SGP,2019,506.0953 +Singapore,SGP,2020,504.61655 +Singapore,SGP,2021,506.1839 +Singapore,SGP,2022,504.21646 +Singapore,SGP,2023,500.87228 +Singapore,SGP,2024,498.74182 +Slovakia,SVK,2000,250.48732 Slovakia,SVK,2001,249.84286 Slovakia,SVK,2002,224.98445 Slovakia,SVK,2003,256.61716 -Slovakia,SVK,2004,252.46227 -Slovakia,SVK,2005,238.67264 +Slovakia,SVK,2004,252.46225 +Slovakia,SVK,2005,238.67262 Slovakia,SVK,2006,230.4 Slovakia,SVK,2007,234.82944 Slovakia,SVK,2008,223.69337 -Slovakia,SVK,2009,220.97957 +Slovakia,SVK,2009,220.97955 Slovakia,SVK,2010,205.0255 -Slovakia,SVK,2011,214.28572 +Slovakia,SVK,2011,214.2857 Slovakia,SVK,2012,203.53358 Slovakia,SVK,2013,181.75438 Slovakia,SVK,2014,172.87137 -Slovakia,SVK,2015,177.31029 +Slovakia,SVK,2015,177.31027 Slovakia,SVK,2016,177.23882 -Slovakia,SVK,2017,183.07803 +Slovakia,SVK,2017,183.07805 Slovakia,SVK,2018,191.67293 Slovakia,SVK,2019,171.21588 Slovakia,SVK,2020,158.54086 Slovakia,SVK,2021,167.34006 Slovakia,SVK,2022,142.04973 -Slovakia,SVK,2023,116.773544 -Slovenia,SVN,1990,375.40195 -Slovenia,SVN,1991,325.7457 -Slovenia,SVN,1992,405.9968 -Slovenia,SVN,1993,419.2469 -Slovenia,SVN,1994,384.0186 -Slovenia,SVN,1995,382.17053 -Slovenia,SVN,1996,367.0498 -Slovenia,SVN,1997,386.19122 -Slovenia,SVN,1998,383.38193 -Slovenia,SVN,1999,361.99097 +Slovakia,SVK,2023,115.137146 +Slovakia,SVK,2024,96.488235 Slovenia,SVN,2000,361.4989 -Slovenia,SVN,2001,367.40332 +Slovenia,SVN,2001,367.40335 Slovenia,SVN,2002,386.30136 Slovenia,SVN,2003,394.79395 Slovenia,SVN,2004,364.1126 Slovenia,SVN,2005,370.61548 Slovenia,SVN,2006,383.85178 -Slovenia,SVN,2007,388.96274 +Slovenia,SVN,2007,388.96277 Slovenia,SVN,2008,350.39612 Slovenia,SVN,2009,341.67175 Slovenia,SVN,2010,353.62854 -Slovenia,SVN,2011,358.66837 +Slovenia,SVN,2011,358.66833 Slovenia,SVN,2012,359.48553 Slovenia,SVN,2013,336.70886 Slovenia,SVN,2014,244.31488 -Slovenia,SVN,2015,316.89188 -Slovenia,SVN,2016,325.9396 +Slovenia,SVN,2015,316.8919 +Slovenia,SVN,2016,325.93964 Slovenia,SVN,2017,318.3801 Slovenia,SVN,2018,304.59055 Slovenia,SVN,2019,298.93015 Slovenia,SVN,2020,277.08704 Slovenia,SVN,2021,269.40344 Slovenia,SVN,2022,257.48505 -Slovenia,SVN,2023,231.27463 +Slovenia,SVN,2023,225.08038 +Slovenia,SVN,2024,227.64705 Solomon Islands,SLB,2000,666.6667 Solomon Islands,SLB,2001,666.6667 Solomon Islands,SLB,2002,666.6667 @@ -4874,169 +4789,166 @@ Solomon Islands,SLB,2018,700 Solomon Islands,SLB,2019,700 Solomon Islands,SLB,2020,700 Solomon Islands,SLB,2021,700 -Solomon Islands,SLB,2022,700 +Solomon Islands,SLB,2022,636.36365 +Solomon Islands,SLB,2023,636.36365 Somalia,SOM,2000,640 Somalia,SOM,2001,653.8462 -Somalia,SOM,2002,666.66675 +Somalia,SOM,2002,666.6667 Somalia,SOM,2003,642.8572 Somalia,SOM,2004,642.8572 -Somalia,SOM,2005,655.17236 +Somalia,SOM,2005,655.1724 Somalia,SOM,2006,666.6666 -Somalia,SOM,2007,645.16125 -Somalia,SOM,2008,656.24994 +Somalia,SOM,2007,645.1613 +Somalia,SOM,2008,656.25 Somalia,SOM,2009,666.6666 Somalia,SOM,2010,666.6666 Somalia,SOM,2011,666.6666 -Somalia,SOM,2012,647.0588 +Somalia,SOM,2012,647.05884 Somalia,SOM,2013,657.1429 Somalia,SOM,2014,657.1429 Somalia,SOM,2015,657.1429 -Somalia,SOM,2016,638.8889 +Somalia,SOM,2016,638.88885 Somalia,SOM,2017,621.62164 Somalia,SOM,2018,621.62164 Somalia,SOM,2019,621.62164 Somalia,SOM,2020,594.5946 -Somalia,SOM,2021,578.9473 -Somalia,SOM,2022,578.9473 +Somalia,SOM,2021,578.9474 +Somalia,SOM,2022,523.8095 +Somalia,SOM,2023,523.8095 South Africa,ZAF,2000,799.4019 South Africa,ZAF,2001,806.2351 South Africa,ZAF,2002,802.285 South Africa,ZAF,2003,807.66766 -South Africa,ZAF,2004,806.9171 -South Africa,ZAF,2005,812.6813 +South Africa,ZAF,2004,806.9172 +South Africa,ZAF,2005,812.6812 South Africa,ZAF,2006,813.48254 -South Africa,ZAF,2007,810.0497 +South Africa,ZAF,2007,810.04974 South Africa,ZAF,2008,809.40027 South Africa,ZAF,2009,807.9019 -South Africa,ZAF,2010,807.5885 -South Africa,ZAF,2011,803.3824 +South Africa,ZAF,2010,807.58856 +South Africa,ZAF,2011,803.3823 South Africa,ZAF,2012,806.84686 South Africa,ZAF,2013,799.13324 -South Africa,ZAF,2014,791.60785 -South Africa,ZAF,2015,789.9361 +South Africa,ZAF,2014,791.6078 +South Africa,ZAF,2015,789.93616 South Africa,ZAF,2016,778.6778 South Africa,ZAF,2017,768.0473 -South Africa,ZAF,2018,774.58746 +South Africa,ZAF,2018,774.5875 South Africa,ZAF,2019,765.6491 -South Africa,ZAF,2020,766.9339 -South Africa,ZAF,2021,744.38214 -South Africa,ZAF,2022,729.67194 -South Africa,ZAF,2023,707.6856 -South America,,2000,160.63931 -South America,,2001,169.65028 -South America,,2002,163.60555 -South America,,2003,162.77438 -South America,,2004,175.22786 -South America,,2005,171.07062 -South America,,2006,170.13005 -South America,,2007,179.00522 -South America,,2008,192.89384 -South America,,2009,183.30548 -South America,,2010,203.23543 -South America,,2011,192.61658 -South America,,2012,220.39978 -South America,,2013,246.32048 -South America,,2014,260.15982 -South America,,2015,259.80664 -South America,,2016,244.62053 -South America,,2017,228.12373 -South America,,2018,215.96355 -South America,,2019,216.02893 -South America,,2020,203.11832 -South America,,2021,220.35345 -South America,,2022,179.66322 +South Africa,ZAF,2020,766.9757 +South Africa,ZAF,2021,744.46405 +South Africa,ZAF,2022,729.72516 +South Africa,ZAF,2023,709.6926 +South Africa,ZAF,2024,708.8762 +South America,,2000,161.7791 +South America,,2001,170.58405 +South America,,2002,164.26143 +South America,,2003,163.88788 +South America,,2004,176.06429 +South America,,2005,171.29742 +South America,,2006,170.79236 +South America,,2007,180.10876 +South America,,2008,194.4562 +South America,,2009,184.45947 +South America,,2010,204.52266 +South America,,2011,193.34859 +South America,,2012,221.86264 +South America,,2013,247.6774 +South America,,2014,259.8892 +South America,,2015,260.91443 +South America,,2016,244.04843 +South America,,2017,229.95987 +South America,,2018,218.28986 +South America,,2019,217.6985 +South America,,2020,204.62964 +South America,,2021,220.38507 +South America,,2022,181.38441 +South America,,2023,171.66669 South Korea,KOR,2000,461.5941 -South Korea,KOR,2001,475.28076 +South Korea,KOR,2001,475.28073 South Korea,KOR,2002,472.78696 South Korea,KOR,2003,457.88885 South Korea,KOR,2004,462.61075 South Korea,KOR,2005,449.55966 -South Korea,KOR,2006,451.17746 -South Korea,KOR,2007,473.6137 +South Korea,KOR,2006,451.1775 +South Korea,KOR,2007,473.61368 South Korea,KOR,2008,477.46326 South Korea,KOR,2009,495.67908 -South Korea,KOR,2010,500.6969 -South Korea,KOR,2011,497.47186 -South Korea,KOR,2012,503.25674 -South Korea,KOR,2013,513.5246 -South Korea,KOR,2014,495.26257 -South Korea,KOR,2015,489.8868 -South Korea,KOR,2016,491.6753 -South Korea,KOR,2017,512.9509 -South Korea,KOR,2018,520.6106 -South Korea,KOR,2019,499.31833 -South Korea,KOR,2020,461.58514 -South Korea,KOR,2021,461.8145 -South Korea,KOR,2022,441.6483 -South Korea,KOR,2023,430.56708 +South Korea,KOR,2010,500.68686 +South Korea,KOR,2011,497.06656 +South Korea,KOR,2012,502.259 +South Korea,KOR,2013,512.29596 +South Korea,KOR,2014,493.6341 +South Korea,KOR,2015,488.11615 +South Korea,KOR,2016,489.88397 +South Korea,KOR,2017,510.88638 +South Korea,KOR,2018,518.283 +South Korea,KOR,2019,496.43832 +South Korea,KOR,2020,457.35724 +South Korea,KOR,2021,459.2767 +South Korea,KOR,2022,438.18445 +South Korea,KOR,2023,432.111 +South Korea,KOR,2024,414.2726 South Sudan,SSD,2012,659.0909 South Sudan,SSD,2013,659.57446 South Sudan,SSD,2014,666.6667 -South Sudan,SSD,2015,655.17236 +South Sudan,SSD,2015,655.1724 South Sudan,SSD,2016,653.8462 -South Sudan,SSD,2017,666.66675 -South Sudan,SSD,2018,654.54553 +South Sudan,SSD,2017,666.6667 +South Sudan,SSD,2018,654.5455 South Sudan,SSD,2019,642.8572 South Sudan,SSD,2020,642.8572 -South Sudan,SSD,2021,637.931 -South Sudan,SSD,2022,629.0322 -Spain,ESP,1990,437.285 -Spain,ESP,1991,427.38028 -Spain,ESP,1992,472.5822 -Spain,ESP,1993,443.97888 -Spain,ESP,1994,433.98462 -Spain,ESP,1995,465.74176 -Spain,ESP,1996,385.71674 -Spain,ESP,1997,430.76193 -Spain,ESP,1998,423.7034 -Spain,ESP,1999,483.34387 +South Sudan,SSD,2021,637.9311 +South Sudan,SSD,2022,611.1111 +South Sudan,SSD,2023,610.16956 Spain,ESP,2000,470.93973 Spain,ESP,2001,422.534 -Spain,ESP,2002,481.9307 +Spain,ESP,2002,481.93073 Spain,ESP,2003,429.26547 Spain,ESP,2004,449.15225 Spain,ESP,2005,475.09158 -Spain,ESP,2006,432.7954 -Spain,ESP,2007,438.87894 -Spain,ESP,2008,390.26434 -Spain,ESP,2009,355.04956 -Spain,ESP,2010,288.24716 -Spain,ESP,2011,337.31216 -Spain,ESP,2012,351.14816 +Spain,ESP,2006,432.79544 +Spain,ESP,2007,438.87897 +Spain,ESP,2008,390.26437 +Spain,ESP,2009,355.04953 +Spain,ESP,2010,288.24713 +Spain,ESP,2011,337.3122 +Spain,ESP,2012,351.14813 Spain,ESP,2013,286.8969 Spain,ESP,2014,289.02493 -Spain,ESP,2015,328.5348 +Spain,ESP,2015,328.53482 Spain,ESP,2016,282.9331 -Spain,ESP,2017,329.2092 +Spain,ESP,2017,329.20917 Spain,ESP,2018,290.8622 Spain,ESP,2019,246.6731 Spain,ESP,2020,199.09816 -Spain,ESP,2021,193.53647 -Spain,ESP,2022,217.76645 -Spain,ESP,2023,174.05005 -Sri Lanka,LKA,2000,358.4071 -Sri Lanka,LKA,2001,361.31937 +Spain,ESP,2021,193.53648 +Spain,ESP,2022,217.1339 +Spain,ESP,2023,169.2753 +Spain,ESP,2024,146.15356 +Sri Lanka,LKA,2000,358.40707 +Sri Lanka,LKA,2001,361.31934 Sri Lanka,LKA,2002,410.6628 Sri Lanka,LKA,2003,382.69986 Sri Lanka,LKA,2004,425.2163 -Sri Lanka,LKA,2005,408.16327 -Sri Lanka,LKA,2006,345.339 -Sri Lanka,LKA,2007,402.83398 -Sri Lanka,LKA,2008,392.13712 -Sri Lanka,LKA,2009,406.4712 -Sri Lanka,LKA,2010,318.39404 -Sri Lanka,LKA,2011,415.078 -Sri Lanka,LKA,2012,495.34286 -Sri Lanka,LKA,2013,300.58774 -Sri Lanka,LKA,2014,373.48422 -Sri Lanka,LKA,2015,418.00156 -Sri Lanka,LKA,2016,521.86176 -Sri Lanka,LKA,2017,533.9674 -Sri Lanka,LKA,2018,433.9378 -Sri Lanka,LKA,2019,506.25787 -Sri Lanka,LKA,2020,501.58527 -Sri Lanka,LKA,2021,403.45648 -Sri Lanka,LKA,2022,509.78134 +Sri Lanka,LKA,2005,408.62653 +Sri Lanka,LKA,2006,345.70517 +Sri Lanka,LKA,2007,403.24216 +Sri Lanka,LKA,2008,392.5328 +Sri Lanka,LKA,2009,406.8826 +Sri Lanka,LKA,2010,319.32773 +Sri Lanka,LKA,2011,415.94455 +Sri Lanka,LKA,2012,496.1832 +Sri Lanka,LKA,2013,301.58728 +Sri Lanka,LKA,2014,461.4148 +Sri Lanka,LKA,2015,418.18182 +Sri Lanka,LKA,2016,522.74316 +Sri Lanka,LKA,2017,533.6022 +Sri Lanka,LKA,2018,448.9415 +Sri Lanka,LKA,2019,518.6075 +Sri Lanka,LKA,2020,513.0382 +Sri Lanka,LKA,2021,418.28574 +Sri Lanka,LKA,2022,509.78137 Sudan,SDN,2000,350.3937 Sudan,SDN,2001,343.28357 Sudan,SDN,2002,373.33334 @@ -5047,227 +4959,226 @@ Sudan,SDN,2006,458.06454 Sudan,SDN,2007,467.0543 Sudan,SDN,2008,482.26953 Sudan,SDN,2009,341.9062 -Sudan,SDN,2010,137.20317 +Sudan,SDN,2010,137.20316 Sudan,SDN,2011,175.08813 -Sudan,SDN,2012,215.78946 +Sudan,SDN,2012,215.78947 Sudan,SDN,2013,147.82608 Sudan,SDN,2014,163.01491 -Sudan,SDN,2015,249.80902 +Sudan,SDN,2015,249.809 Sudan,SDN,2016,303.3631 Sudan,SDN,2017,275.79617 Sudan,SDN,2018,280.4878 Sudan,SDN,2019,254.62393 -Sudan,SDN,2020,247.67441 +Sudan,SDN,2020,247.67442 Sudan,SDN,2021,261.5819 -Sudan,SDN,2022,263.15787 +Sudan,SDN,2022,204.41 +Sudan,SDN,2023,214.32835 Suriname,SUR,2000,187.50002 Suriname,SUR,2001,221.15385 Suriname,SUR,2002,284.55283 Suriname,SUR,2003,300 -Suriname,SUR,2004,321.6783 +Suriname,SUR,2004,321.67834 Suriname,SUR,2005,355.0296 -Suriname,SUR,2006,312.10193 -Suriname,SUR,2007,293.3333 +Suriname,SUR,2006,312.1019 +Suriname,SUR,2007,293.33334 Suriname,SUR,2008,291.39072 Suriname,SUR,2009,300.6536 Suriname,SUR,2010,270.40814 Suriname,SUR,2011,228.75816 -Suriname,SUR,2012,347.30536 -Suriname,SUR,2013,357.95453 -Suriname,SUR,2014,467.033 +Suriname,SUR,2012,347.3054 +Suriname,SUR,2013,357.95456 +Suriname,SUR,2014,467.03296 Suriname,SUR,2015,489.2473 -Suriname,SUR,2016,379.12085 +Suriname,SUR,2016,379.12088 Suriname,SUR,2017,331.49173 -Suriname,SUR,2018,296.08936 -Suriname,SUR,2019,326.20325 -Suriname,SUR,2020,344.82758 +Suriname,SUR,2018,315.7895 +Suriname,SUR,2019,352.9412 +Suriname,SUR,2020,346.34146 Suriname,SUR,2021,349.28232 -Suriname,SUR,2022,349.28232 -Sweden,SWE,1990,33.292236 -Sweden,SWE,1991,40.8927 -Sweden,SWE,1992,46.255054 -Sweden,SWE,1993,51.356194 -Sweden,SWE,1994,56.622284 -Sweden,SWE,1995,53.95198 -Sweden,SWE,1996,78.58616 -Sweden,SWE,1997,51.80618 -Sweden,SWE,1998,49.49934 -Sweden,SWE,1999,49.664173 -Sweden,SWE,2000,44.89431 -Sweden,SWE,2001,46.782173 +Suriname,SUR,2022,374.40762 +Suriname,SUR,2023,383.17755 +Sweden,SWE,2000,44.894306 +Sweden,SWE,2001,46.782177 Sweden,SWE,2002,56.23722 Sweden,SWE,2003,68.542725 Sweden,SWE,2004,48.033783 Sweden,SWE,2005,43.129578 Sweden,SWE,2006,49.69291 -Sweden,SWE,2007,46.02876 +Sweden,SWE,2007,46.028755 Sweden,SWE,2008,47.83189 Sweden,SWE,2009,51.098095 -Sweden,SWE,2010,63.72516 -Sweden,SWE,2011,52.30237 -Sweden,SWE,2012,44.70079 +Sweden,SWE,2010,63.725163 +Sweden,SWE,2011,52.302372 +Sweden,SWE,2012,44.700794 Sweden,SWE,2013,44.69712 Sweden,SWE,2014,40.179737 -Sweden,SWE,2015,40.8642 +Sweden,SWE,2015,40.864197 Sweden,SWE,2016,42.979023 -Sweden,SWE,2017,42.501373 +Sweden,SWE,2017,42.50137 Sweden,SWE,2018,42.72772 Sweden,SWE,2019,42.33464 Sweden,SWE,2020,38.71047 -Sweden,SWE,2021,41.79765 -Sweden,SWE,2022,41.067406 -Sweden,SWE,2023,40.694878 -Switzerland,CHE,2000,32.214157 -Switzerland,CHE,2001,31.949331 -Switzerland,CHE,2002,33.175354 -Switzerland,CHE,2003,33.363945 -Switzerland,CHE,2004,33.8187 -Switzerland,CHE,2005,36.678196 -Switzerland,CHE,2006,34.765816 -Switzerland,CHE,2007,32.967033 -Switzerland,CHE,2008,32.965393 -Switzerland,CHE,2009,32.698368 -Switzerland,CHE,2010,34.519302 -Switzerland,CHE,2011,34.669212 -Switzerland,CHE,2012,34.63966 -Switzerland,CHE,2013,34.06609 -Switzerland,CHE,2014,33.10502 -Switzerland,CHE,2015,34.357502 -Switzerland,CHE,2016,37.806873 -Switzerland,CHE,2017,38.071487 -Switzerland,CHE,2018,34.54518 -Switzerland,CHE,2019,33.867085 -Switzerland,CHE,2020,35.041615 -Switzerland,CHE,2021,37.083992 -Switzerland,CHE,2022,36.13297 -Switzerland,CHE,2023,34.842716 -Syria,SYR,2000,626.5346 -Syria,SYR,2001,632.38586 +Sweden,SWE,2021,41.739433 +Sweden,SWE,2022,41.028603 +Sweden,SWE,2023,38.392212 +Sweden,SWE,2024,35.823833 +Switzerland,CHE,2000,33.575317 +Switzerland,CHE,2001,33.35679 +Switzerland,CHE,2002,34.245525 +Switzerland,CHE,2003,34.435265 +Switzerland,CHE,2004,35.227806 +Switzerland,CHE,2005,38.062286 +Switzerland,CHE,2006,36.21439 +Switzerland,CHE,2007,34.472378 +Switzerland,CHE,2008,34.15871 +Switzerland,CHE,2009,34.048298 +Switzerland,CHE,2010,35.881905 +Switzerland,CHE,2011,36.25954 +Switzerland,CHE,2012,36.254223 +Switzerland,CHE,2013,35.521908 +Switzerland,CHE,2014,34.38927 +Switzerland,CHE,2015,35.568336 +Switzerland,CHE,2016,39.44354 +Switzerland,CHE,2017,39.733997 +Switzerland,CHE,2018,36.053703 +Switzerland,CHE,2019,35.284115 +Switzerland,CHE,2020,36.647686 +Switzerland,CHE,2021,38.516407 +Switzerland,CHE,2022,37.25903 +Switzerland,CHE,2023,38.00712 +Switzerland,CHE,2024,36.599316 +Syria,SYR,2000,626.53467 +Syria,SYR,2001,632.3859 Syria,SYR,2002,663.6979 Syria,SYR,2003,667.7631 -Syria,SYR,2004,630.71185 +Syria,SYR,2004,630.7119 Syria,SYR,2005,634.1463 -Syria,SYR,2006,642.47595 +Syria,SYR,2006,642.476 Syria,SYR,2007,642.61707 -Syria,SYR,2008,655.3824 -Syria,SYR,2009,691.7397 -Syria,SYR,2010,696.3825 +Syria,SYR,2008,655.3823 +Syria,SYR,2009,691.73975 +Syria,SYR,2010,696.38245 Syria,SYR,2011,684.5124 Syria,SYR,2012,676.1698 -Syria,SYR,2013,662.6937 +Syria,SYR,2013,662.69366 Syria,SYR,2014,676.2353 -Syria,SYR,2015,734.5919 +Syria,SYR,2015,734.59186 Syria,SYR,2016,703.90753 -Syria,SYR,2017,708.78516 -Syria,SYR,2018,709.18085 -Syria,SYR,2019,707.4137 -Syria,SYR,2020,701.03723 -Syria,SYR,2021,704.2503 -Syria,SYR,2022,701.6607 -Taiwan,TWN,2000,636.51605 -Taiwan,TWN,2001,654.3603 -Taiwan,TWN,2002,654.3781 -Taiwan,TWN,2003,665.997 -Taiwan,TWN,2004,663.889 -Taiwan,TWN,2005,665.31866 -Taiwan,TWN,2006,669.5679 -Taiwan,TWN,2007,669.69684 -Taiwan,TWN,2008,660.06464 -Taiwan,TWN,2009,657.6831 -Taiwan,TWN,2010,653.18756 -Taiwan,TWN,2011,652.28424 -Taiwan,TWN,2012,649.2651 -Taiwan,TWN,2013,644.03754 -Taiwan,TWN,2014,645.4591 +Syria,SYR,2017,708.7852 +Syria,SYR,2018,699.37274 +Syria,SYR,2019,697.1326 +Syria,SYR,2020,690.9177 +Syria,SYR,2021,692.30774 +Syria,SYR,2022,682.27094 +Taiwan,TWN,2000,636.38824 +Taiwan,TWN,2001,654.2725 +Taiwan,TWN,2002,654.295 +Taiwan,TWN,2003,665.88544 +Taiwan,TWN,2004,663.84326 +Taiwan,TWN,2005,665.2308 +Taiwan,TWN,2006,669.4404 +Taiwan,TWN,2007,669.61456 +Taiwan,TWN,2008,660.02264 +Taiwan,TWN,2009,657.6396 +Taiwan,TWN,2010,653.14716 +Taiwan,TWN,2011,652.23083 +Taiwan,TWN,2012,649.23914 +Taiwan,TWN,2013,643.9979 +Taiwan,TWN,2014,645.44543 Taiwan,TWN,2015,650.18976 -Taiwan,TWN,2016,659.1443 -Taiwan,TWN,2017,684.67395 -Taiwan,TWN,2018,672.0984 +Taiwan,TWN,2016,659.1193 +Taiwan,TWN,2017,684.63696 +Taiwan,TWN,2018,672.1228 Taiwan,TWN,2019,648.96606 Taiwan,TWN,2020,648.0232 -Taiwan,TWN,2021,652.2321 -Taiwan,TWN,2022,639.528 -Taiwan,TWN,2023,642.3775 +Taiwan,TWN,2021,652.029 +Taiwan,TWN,2022,639.26575 +Taiwan,TWN,2023,644.39636 +Taiwan,TWN,2024,635.65106 Tajikistan,TJK,2000,32.30337 Tajikistan,TJK,2001,30.576788 Tajikistan,TJK,2002,28.75817 -Tajikistan,TJK,2003,29.678984 +Tajikistan,TJK,2003,29.678982 Tajikistan,TJK,2004,29.108551 Tajikistan,TJK,2005,27.501463 Tajikistan,TJK,2006,31.305372 Tajikistan,TJK,2007,36.020584 Tajikistan,TJK,2008,35.91331 Tajikistan,TJK,2009,31.637714 -Tajikistan,TJK,2010,24.939173 -Tajikistan,TJK,2011,24.63054 -Tajikistan,TJK,2012,25.928108 +Tajikistan,TJK,2010,24.93917 +Tajikistan,TJK,2011,24.630543 +Tajikistan,TJK,2012,25.92811 Tajikistan,TJK,2013,26.300407 Tajikistan,TJK,2014,32.797028 Tajikistan,TJK,2015,32.051285 -Tajikistan,TJK,2016,56.844547 +Tajikistan,TJK,2016,56.84455 Tajikistan,TJK,2017,74.54445 Tajikistan,TJK,2018,87.63931 -Tajikistan,TJK,2019,98.28244 +Tajikistan,TJK,2019,98.28245 Tajikistan,TJK,2020,109.95025 -Tajikistan,TJK,2021,110.52125 -Tajikistan,TJK,2022,116.85825 +Tajikistan,TJK,2021,110.52124 +Tajikistan,TJK,2022,86.55189 +Tajikistan,TJK,2023,87.5 +Tajikistan,TJK,2024,115.87792 Tanzania,TZA,2000,120.00001 Tanzania,TZA,2001,67.13781 -Tanzania,TZA,2002,62.28374 +Tanzania,TZA,2002,62.283737 Tanzania,TZA,2003,59.04059 Tanzania,TZA,2004,141.76247 Tanzania,TZA,2005,292.34973 -Tanzania,TZA,2006,324.78635 -Tanzania,TZA,2007,233.49057 -Tanzania,TZA,2008,222.47191 +Tanzania,TZA,2006,324.78632 +Tanzania,TZA,2007,233.49059 +Tanzania,TZA,2008,222.47192 Tanzania,TZA,2009,231.25 Tanzania,TZA,2010,255.15948 -Tanzania,TZA,2011,321.0832 +Tanzania,TZA,2011,321.08316 Tanzania,TZA,2012,363.31567 Tanzania,TZA,2013,387.3122 Tanzania,TZA,2014,316.2939 -Tanzania,TZA,2015,370.59726 +Tanzania,TZA,2015,370.59723 Tanzania,TZA,2016,328.44574 Tanzania,TZA,2017,336.63367 -Tanzania,TZA,2018,343.79263 -Tanzania,TZA,2019,336.7609 -Tanzania,TZA,2020,298.08914 -Tanzania,TZA,2021,319.4925 -Tanzania,TZA,2022,339.2461 +Tanzania,TZA,2018,344.73325 +Tanzania,TZA,2019,337.19434 +Tanzania,TZA,2020,298.85056 +Tanzania,TZA,2021,320.23123 +Tanzania,TZA,2022,349.27234 +Tanzania,TZA,2023,371.58472 Thailand,THA,2000,608.459 Thailand,THA,2001,605.5055 Thailand,THA,2002,593.13904 Thailand,THA,2003,591.4266 Thailand,THA,2004,599.2911 Thailand,THA,2005,600.78204 -Thailand,THA,2006,598.5525 +Thailand,THA,2006,598.5524 Thailand,THA,2007,606.8898 Thailand,THA,2008,608.9109 Thailand,THA,2009,605.8945 Thailand,THA,2010,606.3337 -Thailand,THA,2011,600.9784 +Thailand,THA,2011,600.97845 Thailand,THA,2012,597.89294 -Thailand,THA,2013,602.76324 +Thailand,THA,2013,602.7633 Thailand,THA,2014,602.2675 -Thailand,THA,2015,598.0431 +Thailand,THA,2015,598.043 Thailand,THA,2016,596.42975 Thailand,THA,2017,584.22455 Thailand,THA,2018,568.6881 Thailand,THA,2019,563.0797 Thailand,THA,2020,571.3962 Thailand,THA,2021,566.9086 -Thailand,THA,2022,560.7429 -Thailand,THA,2023,549.5827 +Thailand,THA,2022,561.1244 +Thailand,THA,2023,549.8451 +Thailand,THA,2024,554.4976 Togo,TGO,2000,277.77777 -Togo,TGO,2001,384.6154 +Togo,TGO,2001,384.61542 Togo,TGO,2002,312.5 -Togo,TGO,2003,222.22223 +Togo,TGO,2003,222.22221 Togo,TGO,2004,368.42105 -Togo,TGO,2005,421.05264 +Togo,TGO,2005,421.0526 Togo,TGO,2006,409.09094 Togo,TGO,2007,315.78946 Togo,TGO,2008,230.76924 -Togo,TGO,2009,200 +Togo,TGO,2009,199.99998 Togo,TGO,2010,409.09094 Togo,TGO,2011,333.33334 Togo,TGO,2012,468.75003 @@ -5276,11 +5187,12 @@ Togo,TGO,2014,428.57144 Togo,TGO,2015,522.7273 Togo,TGO,2016,521.2766 Togo,TGO,2017,481.48148 -Togo,TGO,2018,406.77963 -Togo,TGO,2019,397.26025 -Togo,TGO,2020,463.76813 -Togo,TGO,2021,447.0588 -Togo,TGO,2022,443.1818 +Togo,TGO,2018,399.99997 +Togo,TGO,2019,459.0164 +Togo,TGO,2020,508.19672 +Togo,TGO,2021,464.78876 +Togo,TGO,2022,477.7778 +Togo,TGO,2023,478.26086 Tonga,TON,2000,750 Tonga,TON,2001,750 Tonga,TON,2002,750 @@ -5298,126 +5210,121 @@ Tonga,TON,2013,600 Tonga,TON,2014,600 Tonga,TON,2015,600 Tonga,TON,2016,666.6667 -Tonga,TON,2017,571.4286 -Tonga,TON,2018,571.4286 -Tonga,TON,2019,571.4286 -Tonga,TON,2020,571.4286 -Tonga,TON,2021,571.4286 +Tonga,TON,2017,571.4285 +Tonga,TON,2018,571.4285 +Tonga,TON,2019,571.4285 +Tonga,TON,2020,571.4285 +Tonga,TON,2021,571.4285 Tonga,TON,2022,625 +Tonga,TON,2023,571.4285 Trinidad and Tobago,TTO,2000,679.4872 -Trinidad and Tobago,TTO,2001,680.851 -Trinidad and Tobago,TTO,2002,680.851 +Trinidad and Tobago,TTO,2001,680.8511 +Trinidad and Tobago,TTO,2002,680.8511 Trinidad and Tobago,TTO,2003,681.677 Trinidad and Tobago,TTO,2004,681.182 Trinidad and Tobago,TTO,2005,680.8511 -Trinidad and Tobago,TTO,2006,680.6137 -Trinidad and Tobago,TTO,2007,679.68744 +Trinidad and Tobago,TTO,2006,680.61365 +Trinidad and Tobago,TTO,2007,679.6875 Trinidad and Tobago,TTO,2008,679.17206 Trinidad and Tobago,TTO,2009,679.8469 Trinidad and Tobago,TTO,2010,681.6038 -Trinidad and Tobago,TTO,2011,681.5068 +Trinidad and Tobago,TTO,2011,681.50684 Trinidad and Tobago,TTO,2012,682.0175 -Trinidad and Tobago,TTO,2013,682.1052 -Trinidad and Tobago,TTO,2014,681.1902 +Trinidad and Tobago,TTO,2013,682.1053 +Trinidad and Tobago,TTO,2014,681.19025 Trinidad and Tobago,TTO,2015,681.2957 -Trinidad and Tobago,TTO,2016,681.5286 +Trinidad and Tobago,TTO,2016,681.5287 Trinidad and Tobago,TTO,2017,681.3305 Trinidad and Tobago,TTO,2018,681.4735 Trinidad and Tobago,TTO,2019,681.5217 Trinidad and Tobago,TTO,2020,681.4735 -Trinidad and Tobago,TTO,2021,681.76917 -Trinidad and Tobago,TTO,2022,681.5286 +Trinidad and Tobago,TTO,2021,681.7691 +Trinidad and Tobago,TTO,2022,682.1053 +Trinidad and Tobago,TTO,2023,682.1053 Tunisia,TUN,2000,586.4023 Tunisia,TUN,2001,586.11597 Tunisia,TUN,2002,584.4595 -Tunisia,TUN,2003,572.695 +Tunisia,TUN,2003,572.69507 Tunisia,TUN,2004,573.3445 -Tunisia,TUN,2005,574.2496 +Tunisia,TUN,2005,574.24963 Tunisia,TUN,2006,581.4843 Tunisia,TUN,2007,585.8807 Tunisia,TUN,2008,582.46344 Tunisia,TUN,2009,577.32635 -Tunisia,TUN,2010,574.57526 +Tunisia,TUN,2010,574.5752 Tunisia,TUN,2011,576.07367 Tunisia,TUN,2012,571.1883 Tunisia,TUN,2013,568.35724 Tunisia,TUN,2014,564.5757 -Tunisia,TUN,2015,567.7618 -Tunisia,TUN,2016,563.42334 -Tunisia,TUN,2017,565.11053 -Tunisia,TUN,2018,563.917 -Tunisia,TUN,2019,560.3843 -Tunisia,TUN,2020,559.19507 -Tunisia,TUN,2021,557.6835 -Tunisia,TUN,2022,564.6226 -Tunisia,TUN,2023,563.95624 -Turkey,TUR,1990,494.78622 -Turkey,TUR,1991,507.30408 -Turkey,TUR,1992,506.75577 -Turkey,TUR,1993,452.51328 -Turkey,TUR,1994,505.04276 -Turkey,TUR,1995,474.95364 -Turkey,TUR,1996,466.4207 -Turkey,TUR,1997,488.67374 -Turkey,TUR,1998,488.2904 -Turkey,TUR,1999,523.9608 -Turkey,TUR,2000,539.97107 -Turkey,TUR,2001,569.2308 -Turkey,TUR,2002,504.59436 -Turkey,TUR,2003,493.65643 -Turkey,TUR,2004,464.90347 -Turkey,TUR,2005,511.23486 -Turkey,TUR,2006,505.5064 -Turkey,TUR,2007,543.3079 -Turkey,TUR,2008,558.0657 -Turkey,TUR,2009,542.5696 -Turkey,TUR,2010,493.72244 -Turkey,TUR,2011,508.72293 -Turkey,TUR,2012,496.28207 -Turkey,TUR,2013,479.5097 -Turkey,TUR,2014,533.3042 -Turkey,TUR,2015,474.39526 -Turkey,TUR,2016,491.3954 -Turkey,TUR,2017,502.09146 -Turkey,TUR,2018,509.74515 -Turkey,TUR,2019,459.6909 -Turkey,TUR,2020,453.9735 -Turkey,TUR,2021,470.99545 -Turkey,TUR,2022,459.10278 -Turkey,TUR,2023,464.5861 +Tunisia,TUN,2015,567.76184 +Tunisia,TUN,2016,563.4234 +Tunisia,TUN,2017,566.08356 +Tunisia,TUN,2018,567.3996 +Tunisia,TUN,2019,564.9409 +Tunisia,TUN,2020,564.5463 +Tunisia,TUN,2021,564.38354 +Tunisia,TUN,2022,565.62354 +Tunisia,TUN,2023,560.357 +Tunisia,TUN,2024,560.24963 +Turkey,TUR,2000,574.1277 +Turkey,TUR,2001,605.28467 +Turkey,TUR,2002,536.4316 +Turkey,TUR,2003,524.7958 +Turkey,TUR,2004,494.22372 +Turkey,TUR,2005,543.5872 +Turkey,TUR,2006,537.4752 +Turkey,TUR,2007,577.79865 +Turkey,TUR,2008,593.53735 +Turkey,TUR,2009,577.0238 +Turkey,TUR,2010,525.04974 +Turkey,TUR,2011,541.00006 +Turkey,TUR,2012,527.7882 +Turkey,TUR,2013,509.97296 +Turkey,TUR,2014,567.25543 +Turkey,TUR,2015,504.6415 +Turkey,TUR,2016,522.777 +Turkey,TUR,2017,534.2931 +Turkey,TUR,2018,541.8624 +Turkey,TUR,2019,488.44028 +Turkey,TUR,2020,482.45032 +Turkey,TUR,2021,500.60995 +Turkey,TUR,2022,487.89618 +Turkey,TUR,2023,493.699 +Turkey,TUR,2024,469.697 Turkmenistan,TKM,2000,1306.5989 Turkmenistan,TKM,2001,1306.3148 Turkmenistan,TKM,2002,1306.542 Turkmenistan,TKM,2003,1306.4814 Turkmenistan,TKM,2004,1306.208 Turkmenistan,TKM,2005,1306.5522 -Turkmenistan,TKM,2006,1306.227 +Turkmenistan,TKM,2006,1306.2272 Turkmenistan,TKM,2007,1306.4517 Turkmenistan,TKM,2008,1306.516 Turkmenistan,TKM,2009,1306.6333 Turkmenistan,TKM,2010,1306.7228 Turkmenistan,TKM,2011,1306.6202 Turkmenistan,TKM,2012,1306.4789 -Turkmenistan,TKM,2013,1306.3062 +Turkmenistan,TKM,2013,1306.3063 Turkmenistan,TKM,2014,1306.3726 Turkmenistan,TKM,2015,1306.7021 Turkmenistan,TKM,2016,1306.7021 -Turkmenistan,TKM,2017,1306.7021 -Turkmenistan,TKM,2018,1306.7021 -Turkmenistan,TKM,2019,1306.7021 -Turkmenistan,TKM,2020,1306.7021 -Turkmenistan,TKM,2021,1306.1224 -Turkmenistan,TKM,2022,1306.0251 +Turkmenistan,TKM,2017,1306.4043 +Turkmenistan,TKM,2018,1306.3881 +Turkmenistan,TKM,2019,1306.6666 +Turkmenistan,TKM,2020,1306.677 +Turkmenistan,TKM,2021,1305.986 +Turkmenistan,TKM,2022,1306.1351 +Turkmenistan,TKM,2023,1306.3036 Turks and Caicos Islands,TCA,2000,625 Turks and Caicos Islands,TCA,2001,625 Turks and Caicos Islands,TCA,2002,700 Turks and Caicos Islands,TCA,2003,636.36365 Turks and Caicos Islands,TCA,2004,666.6667 -Turks and Caicos Islands,TCA,2005,692.3078 +Turks and Caicos Islands,TCA,2005,692.30774 Turks and Caicos Islands,TCA,2006,687.5 Turks and Caicos Islands,TCA,2007,666.6666 Turks and Caicos Islands,TCA,2008,684.2105 -Turks and Caicos Islands,TCA,2009,650 +Turks and Caicos Islands,TCA,2009,649.99994 Turks and Caicos Islands,TCA,2010,666.6667 Turks and Caicos Islands,TCA,2011,666.6667 Turks and Caicos Islands,TCA,2012,666.6667 @@ -5425,12 +5332,13 @@ Turks and Caicos Islands,TCA,2013,666.6667 Turks and Caicos Islands,TCA,2014,652.17395 Turks and Caicos Islands,TCA,2015,652.17395 Turks and Caicos Islands,TCA,2016,640 -Turks and Caicos Islands,TCA,2017,655.17236 +Turks and Caicos Islands,TCA,2017,655.1724 Turks and Caicos Islands,TCA,2018,653.8462 Turks and Caicos Islands,TCA,2019,653.8462 Turks and Caicos Islands,TCA,2020,640 Turks and Caicos Islands,TCA,2021,653.8462 Turks and Caicos Islands,TCA,2022,653.8462 +Turks and Caicos Islands,TCA,2023,653.8462 Uganda,UGA,2000,37.735847 Uganda,UGA,2001,37.735847 Uganda,UGA,2002,34.88372 @@ -5439,60 +5347,50 @@ Uganda,UGA,2004,25.773195 Uganda,UGA,2005,51.02041 Uganda,UGA,2006,181.24998 Uganda,UGA,2007,200 -Uganda,UGA,2008,193.13306 +Uganda,UGA,2008,193.13304 Uganda,UGA,2009,264.4628 Uganda,UGA,2010,291.9708 Uganda,UGA,2011,278.1955 -Uganda,UGA,2012,96.55173 +Uganda,UGA,2012,96.55172 Uganda,UGA,2013,35.83062 -Uganda,UGA,2014,55.72756 +Uganda,UGA,2014,55.727554 Uganda,UGA,2015,56.338028 Uganda,UGA,2016,55.702915 -Uganda,UGA,2017,47.619053 -Uganda,UGA,2018,77.46478 -Uganda,UGA,2019,50.884956 -Uganda,UGA,2020,46.460175 -Uganda,UGA,2021,44.31599 -Uganda,UGA,2022,44.526905 -Ukraine,UKR,1990,575.552 -Ukraine,UKR,1991,561.4285 -Ukraine,UKR,1992,564.7003 -Ukraine,UKR,1993,520.2714 -Ukraine,UKR,1994,504.70178 -Ukraine,UKR,1995,489.71655 -Ukraine,UKR,1996,429.0522 -Ukraine,UKR,1997,417.38303 -Ukraine,UKR,1998,394.4499 -Ukraine,UKR,1999,416.32864 +Uganda,UGA,2017,47.61905 +Uganda,UGA,2018,81.39535 +Uganda,UGA,2019,56.76856 +Uganda,UGA,2020,54.347828 +Uganda,UGA,2021,55.02846 +Uganda,UGA,2022,57.391308 Ukraine,UKR,2000,402.40924 -Ukraine,UKR,2001,408.7709 +Ukraine,UKR,2001,408.77087 Ukraine,UKR,2002,414.124 -Ukraine,UKR,2003,410.6767 +Ukraine,UKR,2003,410.67673 Ukraine,UKR,2004,368.06622 Ukraine,UKR,2005,374.8055 -Ukraine,UKR,2006,404.21463 +Ukraine,UKR,2006,404.21466 Ukraine,UKR,2007,416 -Ukraine,UKR,2008,417.90424 +Ukraine,UKR,2008,417.90427 Ukraine,UKR,2009,409.59622 -Ukraine,UKR,2010,409.80823 -Ukraine,UKR,2011,427.5455 -Ukraine,UKR,2012,441.82425 +Ukraine,UKR,2010,409.80826 +Ukraine,UKR,2011,427.54553 +Ukraine,UKR,2012,441.82422 Ukraine,UKR,2013,448.93762 -Ukraine,UKR,2014,417.62384 +Ukraine,UKR,2014,417.62387 Ukraine,UKR,2015,380.0709 Ukraine,UKR,2016,408.4336 -Ukraine,UKR,2017,347.1546 +Ukraine,UKR,2017,347.15457 Ukraine,UKR,2018,349.2431 Ukraine,UKR,2019,341.50772 Ukraine,UKR,2020,325.39417 -Ukraine,UKR,2021,276.51636 -Ukraine,UKR,2022,259.69302 +Ukraine,UKR,2021,275.6979 +Ukraine,UKR,2022,256.21008 United Arab Emirates,ARE,2000,676.7652 United Arab Emirates,ARE,2001,676.6273 United Arab Emirates,ARE,2002,676.841 United Arab Emirates,ARE,2003,676.64307 United Arab Emirates,ARE,2004,676.77924 -United Arab Emirates,ARE,2005,676.77106 +United Arab Emirates,ARE,2005,676.771 United Arab Emirates,ARE,2006,676.80975 United Arab Emirates,ARE,2007,676.73944 United Arab Emirates,ARE,2008,676.7338 @@ -5501,144 +5399,140 @@ United Arab Emirates,ARE,2010,676.63654 United Arab Emirates,ARE,2011,676.5863 United Arab Emirates,ARE,2012,676.58417 United Arab Emirates,ARE,2013,676.1524 -United Arab Emirates,ARE,2014,675.04724 +United Arab Emirates,ARE,2014,675.0472 United Arab Emirates,ARE,2015,675.22375 United Arab Emirates,ARE,2016,675.25653 United Arab Emirates,ARE,2017,673.2555 -United Arab Emirates,ARE,2018,671.2006 +United Arab Emirates,ARE,2018,671.2007 United Arab Emirates,ARE,2019,660.09393 -United Arab Emirates,ARE,2020,645.08374 -United Arab Emirates,ARE,2021,602.805 -United Arab Emirates,ARE,2022,561.1348 -United Kingdom,GBR,1990,705.01953 -United Kingdom,GBR,1991,700.1362 -United Kingdom,GBR,1992,665.11957 -United Kingdom,GBR,1993,608.214 -United Kingdom,GBR,1994,588.95764 -United Kingdom,GBR,1995,579.4558 -United Kingdom,GBR,1996,556.53094 -United Kingdom,GBR,1997,511.99124 -United Kingdom,GBR,1998,509.58066 -United Kingdom,GBR,1999,492.3265 +United Arab Emirates,ARE,2020,645.69226 +United Arab Emirates,ARE,2021,605.30023 +United Arab Emirates,ARE,2022,556.84564 +United Arab Emirates,ARE,2023,492.69827 United Kingdom,GBR,2000,521.9859 United Kingdom,GBR,2001,528.99133 -United Kingdom,GBR,2002,520.8914 +United Kingdom,GBR,2002,520.8915 United Kingdom,GBR,2003,536.7771 -United Kingdom,GBR,2004,536.02246 +United Kingdom,GBR,2004,536.0225 United Kingdom,GBR,2005,535.25793 United Kingdom,GBR,2006,559.3425 United Kingdom,GBR,2007,556.0596 United Kingdom,GBR,2008,554.3758 -United Kingdom,GBR,2009,504.64484 -United Kingdom,GBR,2010,517.1962 +United Kingdom,GBR,2009,504.64487 +United Kingdom,GBR,2010,517.1963 United Kingdom,GBR,2011,499.63315 United Kingdom,GBR,2012,537.22485 United Kingdom,GBR,2013,506.92194 -United Kingdom,GBR,2014,462.87866 +United Kingdom,GBR,2014,462.87863 United Kingdom,GBR,2015,397.79266 United Kingdom,GBR,2016,333.45126 United Kingdom,GBR,2017,302.19974 -United Kingdom,GBR,2018,284.67526 -United Kingdom,GBR,2019,265.21347 -United Kingdom,GBR,2020,243.57414 -United Kingdom,GBR,2021,268.55664 -United Kingdom,GBR,2022,255.84961 -United Kingdom,GBR,2023,237.58902 -United States,USA,2000,571.1633 -United States,USA,2001,573.3789 -United States,USA,2002,564.7138 -United States,USA,2003,570.05536 -United States,USA,2004,566.3913 -United States,USA,2005,568.222 -United States,USA,2006,559.82544 -United States,USA,2007,562.6556 -United States,USA,2008,555.9448 -United States,USA,2009,531.22217 -United States,USA,2010,536.6583 -United States,USA,2011,517.5635 -United States,USA,2012,500.9169 -United States,USA,2013,501.54474 -United States,USA,2014,497.7576 -United States,USA,2015,474.7367 -United States,USA,2016,454.4961 -United States,USA,2017,441.04965 -United States,USA,2018,435.00613 -United States,USA,2019,413.22342 -United States,USA,2020,387.2002 -United States,USA,2021,398.53186 -United States,USA,2022,385.97964 -United States,USA,2023,369.47318 +United Kingdom,GBR,2018,285.5987 +United Kingdom,GBR,2019,268.31952 +United Kingdom,GBR,2020,246.68149 +United Kingdom,GBR,2021,268.74313 +United Kingdom,GBR,2022,250.76793 +United Kingdom,GBR,2023,228.24567 +United Kingdom,GBR,2024,210.89114 +United States,USA,2000,607.52216 +United States,USA,2001,609.89594 +United States,USA,2002,600.66125 +United States,USA,2003,606.34436 +United States,USA,2004,602.4469 +United States,USA,2005,604.39307 +United States,USA,2006,595.4598 +United States,USA,2007,598.4763 +United States,USA,2008,591.33594 +United States,USA,2009,565.02075 +United States,USA,2010,570.8129 +United States,USA,2011,550.47974 +United States,USA,2012,532.77057 +United States,USA,2013,533.43634 +United States,USA,2014,529.4079 +United States,USA,2015,504.90576 +United States,USA,2016,483.35437 +United States,USA,2017,469.0189 +United States,USA,2018,462.5888 +United States,USA,2019,439.40912 +United States,USA,2020,411.6982 +United States,USA,2021,423.7412 +United States,USA,2022,410.3674 +United States,USA,2023,392.848 +United States,USA,2024,383.5492 United States Virgin Islands,VIR,2000,660.1942 United States Virgin Islands,VIR,2001,660.1942 United States Virgin Islands,VIR,2002,653.8462 United States Virgin Islands,VIR,2003,653.8462 United States Virgin Islands,VIR,2004,657.1429 -United States Virgin Islands,VIR,2005,659.7938 -United States Virgin Islands,VIR,2006,659.7938 +United States Virgin Islands,VIR,2005,659.79376 +United States Virgin Islands,VIR,2006,659.79376 United States Virgin Islands,VIR,2007,653.0612 -United States Virgin Islands,VIR,2008,659.7938 -United States Virgin Islands,VIR,2009,663.0435 +United States Virgin Islands,VIR,2008,659.79376 +United States Virgin Islands,VIR,2009,663.04346 United States Virgin Islands,VIR,2010,656.25 United States Virgin Islands,VIR,2011,663.1579 United States Virgin Islands,VIR,2012,655.1724 United States Virgin Islands,VIR,2013,658.5366 -United States Virgin Islands,VIR,2014,653.8461 -United States Virgin Islands,VIR,2015,653.8461 +United States Virgin Islands,VIR,2014,653.8462 +United States Virgin Islands,VIR,2015,653.8462 United States Virgin Islands,VIR,2016,649.35065 -United States Virgin Islands,VIR,2017,650 -United States Virgin Islands,VIR,2018,645.16125 -United States Virgin Islands,VIR,2019,651.5152 +United States Virgin Islands,VIR,2017,649.99994 +United States Virgin Islands,VIR,2018,645.1613 +United States Virgin Islands,VIR,2019,651.51514 United States Virgin Islands,VIR,2020,640.625 United States Virgin Islands,VIR,2021,636.3636 United States Virgin Islands,VIR,2022,632.35297 -Upper-middle-income countries,,2000,566.1221 -Upper-middle-income countries,,2001,570.22723 -Upper-middle-income countries,,2002,578.80164 -Upper-middle-income countries,,2003,595.6132 -Upper-middle-income countries,,2004,595.59 -Upper-middle-income countries,,2005,604.76746 -Upper-middle-income countries,,2006,615.02905 -Upper-middle-income countries,,2007,624.7782 -Upper-middle-income countries,,2008,610.9768 -Upper-middle-income countries,,2009,615.3201 -Upper-middle-income countries,,2010,613.6308 -Upper-middle-income countries,,2011,626.87915 -Upper-middle-income countries,,2012,613.4987 -Upper-middle-income countries,,2013,615.75385 -Upper-middle-income countries,,2014,608.3646 -Upper-middle-income countries,,2015,590.5744 -Upper-middle-income countries,,2016,580.24536 -Upper-middle-income countries,,2017,577.1386 -Upper-middle-income countries,,2018,573.4566 -Upper-middle-income countries,,2019,563.0966 -Upper-middle-income countries,,2020,551.4689 -Upper-middle-income countries,,2021,548.55963 -Upper-middle-income countries,,2022,538.5725 +United States Virgin Islands,VIR,2023,641.791 +Upper-middle-income countries,,2000,589.51373 +Upper-middle-income countries,,2001,597.01056 +Upper-middle-income countries,,2002,604.2267 +Upper-middle-income countries,,2003,620.6766 +Upper-middle-income countries,,2004,619.2279 +Upper-middle-income countries,,2005,628.1388 +Upper-middle-income countries,,2006,638.1497 +Upper-middle-income countries,,2007,648.1559 +Upper-middle-income countries,,2008,631.01917 +Upper-middle-income countries,,2009,638.2973 +Upper-middle-income countries,,2010,633.89417 +Upper-middle-income countries,,2011,646.51526 +Upper-middle-income countries,,2012,631.7128 +Upper-middle-income countries,,2013,634.2011 +Upper-middle-income countries,,2014,625.7929 +Upper-middle-income countries,,2015,607.29956 +Upper-middle-income countries,,2016,596.53204 +Upper-middle-income countries,,2017,592.25507 +Upper-middle-income countries,,2018,588.7053 +Upper-middle-income countries,,2019,575.83 +Upper-middle-income countries,,2020,566.4339 +Upper-middle-income countries,,2021,562.7759 +Upper-middle-income countries,,2022,551.98206 +Upper-middle-income countries,,2023,547.6743 Uruguay,URY,2000,67.19367 Uruguay,URY,2001,25.945946 Uruguay,URY,2002,27.083332 Uruguay,URY,2003,25.641026 Uruguay,URY,2004,140.67796 -Uruguay,URY,2005,104.166664 -Uruguay,URY,2006,249.55115 +Uruguay,URY,2005,104.16667 +Uruguay,URY,2006,249.55116 Uruguay,URY,2007,108.742004 Uruguay,URY,2008,289.6237 Uruguay,URY,2009,243.36792 Uruguay,URY,2010,111.73185 Uruguay,URY,2011,204.06189 -Uruguay,URY,2012,272.64148 +Uruguay,URY,2012,272.6415 Uruguay,URY,2013,149.22813 Uruguay,URY,2014,85.51618 Uruguay,URY,2015,98.98108 Uruguay,URY,2016,74.98197 Uruguay,URY,2017,64.808365 Uruguay,URY,2018,69.13073 -Uruguay,URY,2019,60.983204 +Uruguay,URY,2019,60.9832 Uruguay,URY,2020,94.32572 Uruguay,URY,2021,150.71921 -Uruguay,URY,2022,112.648224 -Uruguay,URY,2023,128.78789 -Uzbekistan,UZB,2000,1103.4996 +Uruguay,URY,2022,110.363396 +Uruguay,URY,2023,115.68323 +Uruguay,URY,2024,96.69947 +Uzbekistan,UZB,2000,1103.4998 Uzbekistan,UZB,2001,1085.0709 Uzbekistan,UZB,2002,1049.3927 Uzbekistan,UZB,2003,997.33276 @@ -5646,21 +5540,22 @@ Uzbekistan,UZB,2004,1057.9478 Uzbekistan,UZB,2005,984.6864 Uzbekistan,UZB,2006,1105.6366 Uzbekistan,UZB,2007,1114.8511 -Uzbekistan,UZB,2008,1129.8289 +Uzbekistan,UZB,2008,1129.8287 Uzbekistan,UZB,2009,1085.03 -Uzbekistan,UZB,2010,1053.6847 +Uzbekistan,UZB,2010,1053.6848 Uzbekistan,UZB,2011,1118.3712 Uzbekistan,UZB,2012,1098.3019 Uzbekistan,UZB,2013,1125.9843 -Uzbekistan,UZB,2014,1121.0762 +Uzbekistan,UZB,2014,1121.0763 Uzbekistan,UZB,2015,1111.6531 Uzbekistan,UZB,2016,1114.1204 Uzbekistan,UZB,2017,1096.9756 -Uzbekistan,UZB,2018,1139.8823 -Uzbekistan,UZB,2019,1126.5544 -Uzbekistan,UZB,2020,1161.4795 -Uzbekistan,UZB,2021,1163.7942 -Uzbekistan,UZB,2022,1167.6029 +Uzbekistan,UZB,2018,1139.1924 +Uzbekistan,UZB,2019,1119.2573 +Uzbekistan,UZB,2020,1149.1056 +Uzbekistan,UZB,2021,1150.476 +Uzbekistan,UZB,2022,1121.3534 +Uzbekistan,UZB,2023,1121.1803 Vanuatu,VUT,2000,600 Vanuatu,VUT,2001,600 Vanuatu,VUT,2002,600 @@ -5674,63 +5569,66 @@ Vanuatu,VUT,2009,666.6667 Vanuatu,VUT,2010,666.6667 Vanuatu,VUT,2011,666.6667 Vanuatu,VUT,2012,666.6667 -Vanuatu,VUT,2013,571.4286 -Vanuatu,VUT,2014,571.4286 -Vanuatu,VUT,2015,499.99997 -Vanuatu,VUT,2016,571.4286 +Vanuatu,VUT,2013,571.4285 +Vanuatu,VUT,2014,571.4285 +Vanuatu,VUT,2015,500 +Vanuatu,VUT,2016,571.4285 Vanuatu,VUT,2017,666.6667 -Vanuatu,VUT,2018,571.4286 -Vanuatu,VUT,2019,571.4286 -Vanuatu,VUT,2020,571.4286 -Vanuatu,VUT,2021,571.4286 -Vanuatu,VUT,2022,571.4286 +Vanuatu,VUT,2018,500 +Vanuatu,VUT,2019,444.44443 +Vanuatu,VUT,2020,444.44443 +Vanuatu,VUT,2021,500 +Vanuatu,VUT,2022,500 +Vanuatu,VUT,2023,500 Venezuela,VEN,2000,212.73602 Venezuela,VEN,2001,261.56918 Venezuela,VEN,2002,274.14264 Venezuela,VEN,2003,263.7303 -Venezuela,VEN,2004,228.6149 +Venezuela,VEN,2004,228.61491 Venezuela,VEN,2005,211.21548 Venezuela,VEN,2006,207.20883 Venezuela,VEN,2007,217.66922 Venezuela,VEN,2008,215.67476 -Venezuela,VEN,2009,222.1293 +Venezuela,VEN,2009,222.12929 Venezuela,VEN,2010,252.02179 Venezuela,VEN,2011,227.1109 Venezuela,VEN,2012,279.86908 Venezuela,VEN,2013,297.78732 Venezuela,VEN,2014,341.4909 Venezuela,VEN,2015,318.93036 -Venezuela,VEN,2016,356.76413 +Venezuela,VEN,2016,356.76416 Venezuela,VEN,2017,320.80923 -Venezuela,VEN,2018,320.76984 -Venezuela,VEN,2019,320.73257 -Venezuela,VEN,2020,180.27042 -Venezuela,VEN,2021,171.61195 -Venezuela,VEN,2022,185.80202 +Venezuela,VEN,2018,320.9307 +Venezuela,VEN,2019,320.84555 +Venezuela,VEN,2020,184.57333 +Venezuela,VEN,2021,174.24333 +Venezuela,VEN,2022,180.42738 +Venezuela,VEN,2023,180.25078 Vietnam,VNM,2000,321.91266 Vietnam,VNM,2001,290.75464 Vietnam,VNM,2002,344.41342 Vietnam,VNM,2003,375.7635 Vietnam,VNM,2004,407.14285 Vietnam,VNM,2005,462.5576 -Vietnam,VNM,2006,448.3592 +Vietnam,VNM,2006,448.35922 Vietnam,VNM,2007,438.9027 Vietnam,VNM,2008,421.8464 Vietnam,VNM,2009,411.95438 Vietnam,VNM,2010,450.71957 Vietnam,VNM,2011,405.4773 -Vietnam,VNM,2012,369.3991 +Vietnam,VNM,2012,369.39908 Vietnam,VNM,2013,371.36432 -Vietnam,VNM,2014,401.2035 -Vietnam,VNM,2015,470.46533 -Vietnam,VNM,2016,479.431 -Vietnam,VNM,2017,416.12735 -Vietnam,VNM,2018,465.03177 -Vietnam,VNM,2019,539.61835 -Vietnam,VNM,2020,515.8879 -Vietnam,VNM,2021,476.13022 -Vietnam,VNM,2022,409.80072 -Vietnam,VNM,2023,475.44873 +Vietnam,VNM,2014,401.20352 +Vietnam,VNM,2015,483.78928 +Vietnam,VNM,2016,489.14438 +Vietnam,VNM,2017,430.56696 +Vietnam,VNM,2018,482.96344 +Vietnam,VNM,2019,553.97015 +Vietnam,VNM,2020,528.56555 +Vietnam,VNM,2021,475.17844 +Vietnam,VNM,2022,415.48795 +Vietnam,VNM,2023,472.47144 +Vietnam,VNM,2024,471.15857 Western Sahara,ESH,2000,625 Western Sahara,ESH,2001,625 Western Sahara,ESH,2002,666.6666 @@ -5741,96 +5639,100 @@ Western Sahara,ESH,2006,666.6666 Western Sahara,ESH,2007,666.6666 Western Sahara,ESH,2008,666.6666 Western Sahara,ESH,2009,666.6666 -World,OWID_WRL,2000,517.84125 -World,OWID_WRL,2001,520.07886 -World,OWID_WRL,2002,523.3326 -World,OWID_WRL,2003,535.7409 -World,OWID_WRL,2004,530.9171 -World,OWID_WRL,2005,534.97253 -World,OWID_WRL,2006,538.3576 -World,OWID_WRL,2007,546.7014 -World,OWID_WRL,2008,540.01984 -World,OWID_WRL,2009,536.3891 -World,OWID_WRL,2010,536.8582 -World,OWID_WRL,2011,543.61914 -World,OWID_WRL,2012,542.0042 -World,OWID_WRL,2013,542.36743 -World,OWID_WRL,2014,538.8367 -World,OWID_WRL,2015,527.64355 -World,OWID_WRL,2016,520.14777 -World,OWID_WRL,2017,516.63275 -World,OWID_WRL,2018,512.9673 -World,OWID_WRL,2019,500.49585 -World,OWID_WRL,2020,487.3561 -World,OWID_WRL,2021,490.35458 -World,OWID_WRL,2022,485.99475 -World,OWID_WRL,2023,480.84857 +World,OWID_WRL,2000,527.2703 +World,OWID_WRL,2001,529.2534 +World,OWID_WRL,2002,532.3002 +World,OWID_WRL,2003,544.55054 +World,OWID_WRL,2004,539.43054 +World,OWID_WRL,2005,543.3786 +World,OWID_WRL,2006,546.40344 +World,OWID_WRL,2007,554.659 +World,OWID_WRL,2008,547.7068 +World,OWID_WRL,2009,543.47864 +World,OWID_WRL,2010,543.83795 +World,OWID_WRL,2011,550.17773 +World,OWID_WRL,2012,548.08185 +World,OWID_WRL,2013,548.2747 +World,OWID_WRL,2014,544.6569 +World,OWID_WRL,2015,533.20233 +World,OWID_WRL,2016,525.1786 +World,OWID_WRL,2017,521.9015 +World,OWID_WRL,2018,518.415 +World,OWID_WRL,2019,505.46906 +World,OWID_WRL,2020,491.96887 +World,OWID_WRL,2021,494.70203 +World,OWID_WRL,2022,489.7454 +World,OWID_WRL,2023,484.40286 +World,OWID_WRL,2024,473.0065 Yemen,YEM,2000,656.8915 -Yemen,YEM,2001,659.3407 +Yemen,YEM,2001,659.34064 Yemen,YEM,2002,657.82495 Yemen,YEM,2003,658.5366 -Yemen,YEM,2004,658.2569 +Yemen,YEM,2004,658.25684 Yemen,YEM,2005,658.28094 -Yemen,YEM,2006,658.62714 +Yemen,YEM,2006,658.6271 Yemen,YEM,2007,658.37476 Yemen,YEM,2008,658.01526 Yemen,YEM,2009,657.7778 -Yemen,YEM,2010,703.6083 +Yemen,YEM,2010,703.6082 Yemen,YEM,2011,694.0419 Yemen,YEM,2012,711.45685 -Yemen,YEM,2013,712.9412 +Yemen,YEM,2013,712.94116 Yemen,YEM,2014,723.2376 Yemen,YEM,2015,713.5417 Yemen,YEM,2016,694.38666 Yemen,YEM,2017,669.1358 -Yemen,YEM,2018,590.2578 +Yemen,YEM,2018,590.2579 Yemen,YEM,2019,585.36584 Yemen,YEM,2020,578.2313 -Yemen,YEM,2021,572.8477 -Yemen,YEM,2022,566.1016 +Yemen,YEM,2021,589.0411 +Yemen,YEM,2022,584.1584 +Yemen,YEM,2023,586.3192 Zambia,ZMB,2000,27.237352 -Zambia,ZMB,2001,28.967253 +Zambia,ZMB,2001,28.967255 Zambia,ZMB,2002,29.411764 Zambia,ZMB,2003,30.048079 Zambia,ZMB,2004,28.202114 Zambia,ZMB,2005,29.082775 -Zambia,ZMB,2006,24.793388 -Zambia,ZMB,2007,24.896263 -Zambia,ZMB,2008,25.183632 +Zambia,ZMB,2006,24.793386 +Zambia,ZMB,2007,24.896265 +Zambia,ZMB,2008,25.18363 Zambia,ZMB,2009,24.266935 Zambia,ZMB,2010,24.880383 Zambia,ZMB,2011,24.369017 Zambia,ZMB,2012,24.252224 Zambia,ZMB,2013,24.81203 Zambia,ZMB,2014,42.699722 -Zambia,ZMB,2015,43.67135 +Zambia,ZMB,2015,43.671352 Zambia,ZMB,2016,65.420555 Zambia,ZMB,2017,125 -Zambia,ZMB,2018,140.57704 -Zambia,ZMB,2019,138.5135 +Zambia,ZMB,2018,140.66338 +Zambia,ZMB,2019,138.60716 Zambia,ZMB,2020,135.25935 -Zambia,ZMB,2021,86.907455 +Zambia,ZMB,2021,86.90745 Zambia,ZMB,2022,111.96713 +Zambia,ZMB,2023,110.996925 Zimbabwe,ZWE,2000,476.39487 Zimbabwe,ZWE,2001,544.30383 Zimbabwe,ZWE,2002,487.17947 Zimbabwe,ZWE,2003,351.53583 Zimbabwe,ZWE,2004,384.77365 -Zimbabwe,ZWE,2005,419.8718 -Zimbabwe,ZWE,2006,294.48618 +Zimbabwe,ZWE,2005,419.87183 +Zimbabwe,ZWE,2006,294.4862 Zimbabwe,ZWE,2007,261.8421 Zimbabwe,ZWE,2008,226.73657 Zimbabwe,ZWE,2009,226.33745 Zimbabwe,ZWE,2010,292.82407 Zimbabwe,ZWE,2011,377.31735 -Zimbabwe,ZWE,2012,357.37704 +Zimbabwe,ZWE,2012,357.37708 Zimbabwe,ZWE,2013,410.9015 Zimbabwe,ZWE,2014,396.55176 -Zimbabwe,ZWE,2015,420.2294 +Zimbabwe,ZWE,2015,420.22943 Zimbabwe,ZWE,2016,488.14505 Zimbabwe,ZWE,2017,403.97348 -Zimbabwe,ZWE,2018,395.12714 -Zimbabwe,ZWE,2019,428.7397 +Zimbabwe,ZWE,2018,395.5461 +Zimbabwe,ZWE,2019,429.24533 Zimbabwe,ZWE,2020,371.08792 -Zimbabwe,ZWE,2021,270.68558 -Zimbabwe,ZWE,2022,297.87234 \ No newline at end of file +Zimbabwe,ZWE,2021,269.54492 +Zimbabwe,ZWE,2022,297.53918 +Zimbabwe,ZWE,2023,298.4356 \ No newline at end of file diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go index 8174101f..a16c7c0f 100644 --- a/pkg/k8s/client.go +++ b/pkg/k8s/client.go @@ -8,14 +8,18 @@ import ( "fmt" "log/slog" "os" + "sync" "time" "google.golang.org/grpc" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" podresourcesapi "k8s.io/kubelet/pkg/apis/podresources/v1" @@ -31,12 +35,19 @@ const ( grpcClientRecvSizeMax = 128 * 1024 * 1024 ) +var podMu = sync.RWMutex{} + type Client struct { Logger *slog.Logger Hostname string Config *rest.Config Clientset kubernetes.Interface PodResourceClient podresourcesapi.PodResourcesListerClient + PodInformer coreinformers.PodInformer + informerFactory informers.SharedInformerFactory + stopCh chan struct{} + informerRunning bool + pods map[string]*v1.Pod cleanup func() error } @@ -98,33 +109,32 @@ func New(kubeconfigPath string, kubeletSocket string, logger *slog.Logger) (*Cli return nil, err } - // If kubelet socket is mounted, create a pod resource client - var client podresourcesapi.PodResourcesListerClient - - var cleanup func() error + // Make a new instance of client + c := &Client{ + Logger: logger, + Hostname: os.Getenv(nodenameEnvVar), + Config: config, + Clientset: clientset, + pods: make(map[string]*v1.Pod), + stopCh: make(chan struct{}), + } + // If kubelet socket is mounted, create a pod resource client if _, err := os.Stat(kubeletSocket); err == nil { conn, err := ConnectToServer(kubeletSocket) if err != nil { return nil, err } - client = podresourcesapi.NewPodResourcesListerClient(conn) + c.PodResourceClient = podresourcesapi.NewPodResourcesListerClient(conn) // Close connection when stopping client - cleanup = func() error { + c.cleanup = func() error { return conn.Close() } } - return &Client{ - Logger: logger, - Hostname: os.Getenv(nodenameEnvVar), - Config: config, - Clientset: clientset, - PodResourceClient: client, - cleanup: cleanup, - }, nil + return c, nil } // Close stops clients and release resources. @@ -135,11 +145,75 @@ func (c *Client) Close() error { return c.cleanup() } + // Stop informer(s) + if c.informerRunning { + close(c.stopCh) + } + return nil } -// Pods returns current pods in the cluster. -func (c *Client) Pods(ctx context.Context, ns string, opts metav1.ListOptions) ([]Pod, error) { +// NewInformer creates new pod informer using current client. +func (c *Client) NewPodInformer(resyncPeriod time.Duration) error { + // Create a new informer from factory + // Discussion on ideal resync period: https://groups.google.com/g/kubernetes-sig-api-machinery/c/PbSCXdLDno0 + c.informerFactory = informers.NewSharedInformerFactory(c.Clientset, resyncPeriod) + + // Create a new instance of pod informer + c.PodInformer = c.informerFactory.Core().V1().Pods() + if _, err := c.PodInformer.Informer().AddEventHandler( + // Your custom resource event handlers. + cache.ResourceEventHandlerFuncs{ + // Called on creation + AddFunc: c.podAdd, + // Called on resource update and every resyncPeriod on existing resources. + UpdateFunc: c.podUpdate, + // Called on resource deletion. + DeleteFunc: c.podDelete, + }, + ); err != nil { + return err + } + + return nil +} + +// StartInformer starts shared informers and waits for the shared informer cache to +// synchronize. +func (c *Client) StartInformer() error { + // Starts all the shared informers that have been created by the factory so + // far. + c.informerFactory.Start(c.stopCh) + + // wait for the initial synchronization of the local cache. + if !cache.WaitForCacheSync(c.stopCh, c.PodInformer.Informer().HasSynced) { + return errors.New("failed to synchronize pod informer") + } + + // Set informerRunning to true + c.informerRunning = true + + return nil +} + +// Pods returns a slice of pods provided by shared informer. +func (c *Client) Pods() []*v1.Pod { + var pods []*v1.Pod + + podMu.Lock() + for _, pod := range c.pods { + pods = append(pods, pod) + } + + // Reset pods map + c.pods = make(map[string]*v1.Pod) + podMu.Unlock() + + return pods +} + +// ListPods returns lists all current pods in the cluster. +func (c *Client) ListPods(ctx context.Context, ns string, opts metav1.ListOptions) ([]Pod, error) { resp, err := c.Clientset.CoreV1().Pods(ns).List(ctx, opts) if err != nil { return nil, fmt.Errorf("failed to get pods: %w", err) @@ -163,8 +237,8 @@ func (c *Client) Pods(ctx context.Context, ns string, opts metav1.ListOptions) ( return pods, nil } -// PodDevices returns a slice of pods with devices associated with each pod. -func (c *Client) PodDevices(ctx context.Context) ([]Pod, error) { +// ListPodsWithDevs returns a slice of pods with devices associated with each pod. +func (c *Client) ListPodsWithDevs(ctx context.Context) ([]Pod, error) { if c.PodResourceClient == nil { return nil, errors.New("pod resource API is not available") } @@ -178,7 +252,7 @@ func (c *Client) PodDevices(ctx context.Context) ([]Pod, error) { } // Get pods from all namespaces on the current node - pods, err := c.Pods(ctx, "", opts) + pods, err := c.ListPods(ctx, "", opts) if err != nil { return nil, err } @@ -195,6 +269,27 @@ func (c *Client) PodDevices(ctx context.Context) ([]Pod, error) { return mergePodResources(pods, resp), nil } +// ListUsers returns a slice of map of namespaces to users fetched from rolebindings. +func (c *Client) ListUsers(ctx context.Context, ns string) (map[string][]string, error) { + resp, err := c.Clientset.RbacV1().RoleBindings(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get role bindings: %w", err) + } + + // Loop over role bindings and make a map of namespaces to users + nsUsers := make(map[string][]string) + + for _, bind := range resp.Items { + for _, sub := range bind.Subjects { + if sub.Kind == "User" { + nsUsers[bind.Namespace] = append(nsUsers[bind.Namespace], sub.Name) + } + } + } + + return nsUsers, nil +} + // Exec executes a given command in the pod and returns output. func (c *Client) Exec(ctx context.Context, ns string, pod string, container string, cmd []string) ([]byte, []byte, error) { req := c.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(pod).Namespace(ns).SubResource("exec") @@ -236,6 +331,43 @@ func (c *Client) Exec(ctx context.Context, ns string, pod string, container stri return stdout.Bytes(), stderr.Bytes(), nil } +// ConfigMap returns contents of a configMap. +func (c *Client) ConfigMap(ctx context.Context, ns string, name string) (map[string]string, error) { + resp, err := c.Clientset.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get configmap: %w", err) + } + + return resp.Data, nil +} + +// podAdd adds pod to pods map. +func (c *Client) podAdd(obj any) { + if pod, ok := obj.(*v1.Pod); ok { + podMu.Lock() + c.pods[string(pod.UID)] = pod + podMu.Unlock() + } +} + +// podUpdate updates pod in pods map. +func (c *Client) podUpdate(_, newObj any) { + if pod, ok := newObj.(*v1.Pod); ok { + podMu.Lock() + c.pods[string(pod.UID)] = pod + podMu.Unlock() + } +} + +// podDelete deletes pod from pods map. +func (c *Client) podDelete(obj any) { + if pod, ok := obj.(*v1.Pod); ok { + podMu.Lock() + c.pods[string(pod.UID)] = pod + podMu.Unlock() + } +} + // mergePodResources merges the existing Pod resources, if and when found, to Pod struct. func mergePodResources(pods []Pod, resp *podresourcesapi.ListPodResourcesResponse) []Pod { // Make a map of pods for easy lookup diff --git a/pkg/k8s/client_test.go b/pkg/k8s/client_test.go index f4c4e8b1..8afc6630 100644 --- a/pkg/k8s/client_test.go +++ b/pkg/k8s/client_test.go @@ -10,10 +10,12 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" @@ -25,38 +27,105 @@ import ( var noOpLogger = slog.New(slog.DiscardHandler) -var testPods = []runtime.Object{ - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod11", - UID: "uid11", - Namespace: "ns1", - Labels: map[string]string{ - "label1": "value1", +var ( + testPods = []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod11", + UID: "uid11", + Namespace: "ns1", + Labels: map[string]string{ + "label1": "value1", + }, }, }, - }, - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod12", - UID: "uid12", - Namespace: "ns1", - Labels: map[string]string{ - "label1": "value1", + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod12", + UID: "uid12", + Namespace: "ns1", + Labels: map[string]string{ + "label1": "value1", + }, }, }, - }, - &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod21", - UID: "uid21", - Namespace: "ns2", - Labels: map[string]string{ - "label2": "value2", + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod21", + UID: "uid21", + Namespace: "ns2", + Labels: map[string]string{ + "label2": "value2", + }, }, }, - }, -} + } + + testRoleBindings = []runtime.Object{ + &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rb1", + UID: "rb1", + Namespace: "foo", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "foo", + Namespace: "foo", + }, + { + Kind: "User", + Name: "fooplus", + Namespace: "foo", + }, + { + Kind: "Group", + Name: "foogroup", + Namespace: "foo", + }, + { + Kind: "ServiceAccount", + Name: "foosvc", + Namespace: "foo", + }, + }, + }, + &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rb2", + UID: "rb2", + Namespace: "foo", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "bar", + Namespace: "foo", + }, + }, + }, + &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rb3", + UID: "rb3", + Namespace: "bar", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "barplus", + Namespace: "bar", + }, + { + Kind: "User", + Name: "fooplus", + Namespace: "bar", + }, + }, + }, + } +) func TestNew(t *testing.T) { content := ` @@ -98,7 +167,7 @@ users: require.NoError(t, err) } -func TestPods(t *testing.T) { +func TestPodsLister(t *testing.T) { testCases := []struct { name string targetNS string @@ -166,13 +235,46 @@ func TestPods(t *testing.T) { } for _, test := range testCases { - got, err := client.Pods(t.Context(), test.targetNS, test.opts) + got, err := client.ListPods(t.Context(), test.targetNS, test.opts) require.NoError(t, err) assert.ElementsMatch(t, test.expected, got, test.name) } } +func TestInformer(t *testing.T) { + // Make fake client + fakeClientset := fake.NewClientset(testPods...) + + // Make k8s client + client := &Client{ + Logger: noOpLogger, + Clientset: fakeClientset, + pods: make(map[string]*v1.Pod), + stopCh: make(chan struct{}), + } + + // Make new informer + err := client.NewPodInformer(time.Second) + require.NoError(t, err) + + // Start informer + err = client.StartInformer() + require.NoError(t, err) + + // Sleep a while to fetch all pods + time.Sleep(time.Second) + + // Fetch current pods + pods := client.Pods() + + // Check fetched pods + assert.Len(t, pods, len(testPods)) + + // Stop informer + client.Close() +} + func TestPodDevices(t *testing.T) { podResourcesResp := &podresourcesapi.ListPodResourcesResponse{ PodResources: []*podresourcesapi.PodResources{ @@ -292,7 +394,7 @@ func TestPodDevices(t *testing.T) { PodResourceClient: kubeletClient, } - got, err := client.PodDevices(t.Context()) + got, err := client.ListPodsWithDevs(t.Context()) require.NoError(t, err) assert.ElementsMatch(t, expected, got) @@ -343,3 +445,69 @@ GPU 3: NVIDIA H100 80GB HBM3 (UUID: GPU-2114ac3c-d010-ef91-2ab8-45544c7b64c5)` assert.Equal(t, expected, string(stdout)) assert.Empty(t, stderr) } + +func TestListUsers(t *testing.T) { + // Make fake client + fakeClientset := fake.NewClientset(testRoleBindings...) + + // Make k8s client + client := &Client{ + Logger: noOpLogger, + Clientset: fakeClientset, + pods: make(map[string]*v1.Pod), + stopCh: make(chan struct{}), + } + + // Fetch users + usersMap, err := client.ListUsers(t.Context(), "") + require.NoError(t, err) + + // Expected + expectedUsersMap := map[string][]string{ + "foo": {"foo", "fooplus", "bar"}, + "bar": {"barplus", "fooplus"}, + } + + // Check fetched pods + assert.Equal(t, expectedUsersMap, usersMap) +} + +func TestConfigMap(t *testing.T) { + // Expected data + expectedData := map[string]string{ + "test": "value", + } + + // Make fake client + fakeClientset := fake.NewClientset( + []runtime.Object{ + &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + UID: "cm", + Namespace: "default", + }, + Data: expectedData, + }, + }..., + ) + + // Make k8s client + client := &Client{ + Logger: noOpLogger, + Clientset: fakeClientset, + pods: make(map[string]*v1.Pod), + stopCh: make(chan struct{}), + } + + // Fetch config + cm, err := client.ConfigMap(t.Context(), "default", "cm") + require.NoError(t, err) + + // Check config + assert.Equal(t, expectedData, cm) + + // Try to fetch non existent cm + _, err = client.ConfigMap(t.Context(), "foo", "bar") + require.Error(t, err) +} diff --git a/pkg/lb/backend/tsdb_test.go b/pkg/lb/backend/tsdb_test.go index b170238e..728568cf 100644 --- a/pkg/lb/backend/tsdb_test.go +++ b/pkg/lb/backend/tsdb_test.go @@ -26,17 +26,17 @@ func testTSDBServer(storageRetention string, emptyResponse bool, basicAuth bool) } expectedRange := tsdb.Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "resultType": "matrix", - "result": []interface{}{ - map[string]interface{}{ + "result": []any{ + map[string]any{ "metric": map[string]string{ "__name__": "up", "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{time.Now().Add(-15 * 24 * time.Hour).Unix(), "1"}, - []interface{}{time.Now().Add(-15 * 23 * time.Hour).Unix(), "1"}, + "values": []any{ + []any{time.Now().Add(-15 * 24 * time.Hour).Unix(), "1"}, + []any{time.Now().Add(-15 * 23 * time.Hour).Unix(), "1"}, }, }, }, diff --git a/pkg/lb/cli/cli.go b/pkg/lb/cli/cli.go index 26516510..29c12294 100644 --- a/pkg/lb/cli/cli.go +++ b/pkg/lb/cli/cli.go @@ -91,7 +91,7 @@ func (c *CEEMSLBAppConfig) Validate() error { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -func (c *CEEMSLBAppConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (c *CEEMSLBAppConfig) UnmarshalYAML(unmarshal func(any) error) error { // Set a default config *c = CEEMSLBAppConfig{ CEEMSLBConfig{ diff --git a/pkg/lb/frontend/frontend_test.go b/pkg/lb/frontend/frontend_test.go index 0e29c96e..dc0a9f51 100644 --- a/pkg/lb/frontend/frontend_test.go +++ b/pkg/lb/frontend/frontend_test.go @@ -71,7 +71,7 @@ func dummyTSDBServer(clusterID string) *httptest.Server { expectedFlags := tsdb.Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "query.lookback-delta": "5m", "query.max-samples": "50000000", "query.timeout": "2m", diff --git a/pkg/lb/frontend/parse.go b/pkg/lb/frontend/parse.go index 3462db9c..a7faa9fc 100644 --- a/pkg/lb/frontend/parse.go +++ b/pkg/lb/frontend/parse.go @@ -141,7 +141,7 @@ func parseReqParams(p *ReqParams, req string) { // Extract UUIDs from query for _, match := range regexpUUID.FindAllStringSubmatch(req, -1) { if len(match) > 1 { - for _, uuid := range strings.Split(match[1], "|") { + for uuid := range strings.SplitSeq(match[1], "|") { // Ignore empty strings if strings.TrimSpace(uuid) != "" && !slices.Contains(p.uuids, uuid) { p.uuids = append(p.uuids, uuid) @@ -155,7 +155,7 @@ func parseReqParams(p *ReqParams, req string) { idMatches := regexID.FindAllStringSubmatch(req, -1) for _, match := range idMatches { if len(match) > 1 { - for _, idMatch := range strings.Split(match[1], "|") { + for idMatch := range strings.SplitSeq(match[1], "|") { // Ignore empty strings if strings.TrimSpace(idMatch) != "" { p.clusterID = strings.TrimSpace(idMatch) diff --git a/pkg/sqlite3/sqlite3.go b/pkg/sqlite3/sqlite3.go index 05e18d15..817d60fd 100644 --- a/pkg/sqlite3/sqlite3.go +++ b/pkg/sqlite3/sqlite3.go @@ -20,6 +20,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "maps" "sync" "github.com/mahendrapaipuri/ceems/pkg/api/models" @@ -171,9 +172,7 @@ func addMetricMap(existing, current string) string { // Make a deep copy of existingMetricMap into updatedMetricMap updatedMetricMap := make(models.MetricMap) - for metricName, metricValue := range existingMetricMap { - updatedMetricMap[metricName] = metricValue - } + maps.Copy(updatedMetricMap, existingMetricMap) // Walk through new map and update existing with new. for metricName, newMetricValue := range currentMetricMap { diff --git a/pkg/tsdb/client.go b/pkg/tsdb/client.go index 4450621b..705f1cc5 100644 --- a/pkg/tsdb/client.go +++ b/pkg/tsdb/client.go @@ -45,8 +45,8 @@ type Config struct { type Result struct { Metric map[string]string `json:"metric"` - Value interface{} `json:"value"` - Values []interface{} `json:"values"` + Value any `json:"value"` + Values []any `json:"values"` } type Data struct { @@ -383,7 +383,7 @@ func (t *Client) fetchSettings(ctx context.Context) (*Settings, error) { if retentionPeriod, err = model.ParseDuration(info.StorageRetention); err != nil { // If storageRetention is set to size or time and size, we need to get // "actual" retention period - for _, retentionString := range strings.Split(info.StorageRetention, "or") { + for retentionString := range strings.SplitSeq(info.StorageRetention, "or") { retentionPeriod, err = model.ParseDuration(strings.TrimSpace(retentionString)) if err != nil { continue diff --git a/pkg/tsdb/client_test.go b/pkg/tsdb/client_test.go index dfbea6a2..cfea2b64 100644 --- a/pkg/tsdb/client_test.go +++ b/pkg/tsdb/client_test.go @@ -37,7 +37,7 @@ func testTSDBServer(emptyResponse bool) *httptest.Server { expectedFlags := Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "query.lookback-delta": "5m", "query.max-samples": "50000000", "query.timeout": "2m", @@ -46,7 +46,7 @@ func testTSDBServer(emptyResponse bool) *httptest.Server { expectedRuntimeInfo := Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "startTime": "2025-02-18T07:43:52.775090028Z", "CWD": "/var/lib/prometheus", "reloadConfigSuccess": true, @@ -91,22 +91,22 @@ func testTSDBServer(emptyResponse bool) *httptest.Server { expectedQuery := Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "resultType": "vector", - "result": []interface{}{ - map[string]interface{}{ + "result": []any{ + map[string]any{ "metric": map[string]string{ "uuid": "1", }, - "value": []interface{}{ + "value": []any{ 12345, "1.1", }, }, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": "2", }, - "value": []interface{}{ + "value": []any{ 12345, "2.2", }, }, @@ -116,17 +116,17 @@ func testTSDBServer(emptyResponse bool) *httptest.Server { expectedQueryRange := Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "resultType": "matrix", - "result": []interface{}{ - map[string]interface{}{ + "result": []any{ + map[string]any{ "metric": map[string]string{ "__name__": "up", "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{1727367964.929, "1"}, - []interface{}{1727368964.929, "1"}, + "values": []any{ + []any{1727367964.929, "1"}, + []any{1727368964.929, "1"}, }, }, }, diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 3a9af967..aa19299d 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -159,6 +159,10 @@ then then desc="/projects/admin end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-project-admin-query.txt' + elif [ "${scenario}" = "api-project-query-k8s" ] + then + desc="/projects end point test with k8s" + fixture='pkg/api/testdata/output/e2e-test-api-server-project-query-k8s.txt' elif [ "${scenario}" = "api-user-query" ] then desc="/users end point test" @@ -171,6 +175,10 @@ then then desc="/users/admin end point test that queries all users" fixture='pkg/api/testdata/output/e2e-test-api-server-user-admin-all-query.txt' + elif [ "${scenario}" = "api-user-query-k8s" ] + then + desc="/users end point test with k8s" + fixture='pkg/api/testdata/output/e2e-test-api-server-user-query-k8s.txt' elif [ "${scenario}" = "api-cluster-admin-query" ] then desc="/clusters/admin end point test" @@ -187,6 +195,10 @@ then then desc="/units end point test with running query param" fixture='pkg/api/testdata/output/e2e-test-api-server-running-query.txt' + elif [ "${scenario}" = "api-units-query-k8s" ] + then + desc="/units end point test with k8s" + fixture='pkg/api/testdata/output/e2e-test-api-server-query-k8s.txt' elif [ "${scenario}" = "api-admin-query" ] then desc="/units/admin end point test for admin query" @@ -223,6 +235,10 @@ then then desc="/usage/current/admin end point test with experimental aggregation" fixture='pkg/api/testdata/output/e2e-test-api-server-current-usage-admin-experimental-query.txt' + elif [ "${scenario}" = "api-current-usage-query-k8s" ] + then + desc="/usage/current end point test with k8s" + fixture='pkg/api/testdata/output/e2e-test-api-server-current-usage-query-k8s.txt' elif [ "${scenario}" = "api-global-usage-admin-query" ] then desc="/usage/global/admin end point test" @@ -239,6 +255,10 @@ then then desc="/stats/global/admin end point test" fixture='pkg/api/testdata/output/e2e-test-api-server-global-stats-admin-query.txt' + elif [ "${scenario}" = "api-global-usage-query-k8s" ] + then + desc="/usage/global end point test with k8s" + fixture='pkg/api/testdata/output/e2e-test-api-server-global-usage-query-k8s.txt' elif [ "${scenario}" = "api-verify-pass-query" ] then desc="/units/verify end point test with pass request" @@ -852,12 +872,13 @@ then fi export PATH="${GOBIN:-}:${PATH}" - ./bin/mock_servers prom os-compute os-identity >> "${logfile}" 2>&1 & + ./bin/mock_servers prom os-compute os-identity k8s-api >> "${logfile}" 2>&1 & MOCK_SERVERS_PID=$! waitport "9090" waitport "8080" waitport "7070" + waitport "9080" # Copy config file to tmpdir cp pkg/api/testdata/config.yml "${tmpdir}/config.yml" @@ -892,6 +913,9 @@ then elif [ "${scenario}" = "api-project-admin-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/projects/admin?project=test-project-3" > "${fixture_output}" + elif [ "${scenario}" = "api-project-query-k8s" ] + then + get -H "X-Grafana-User: rb1" "127.0.0.1:${port}/api/${api_version}/projects?project=ns2" > "${fixture_output}" elif [ "${scenario}" = "api-user-query" ] then get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/users" > "${fixture_output}" @@ -901,6 +925,9 @@ then elif [ "${scenario}" = "api-user-admin-all-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/users/admin" > "${fixture_output}" + elif [ "${scenario}" = "api-user-query-k8s" ] + then + get -H "X-Grafana-User: rb1" "127.0.0.1:${port}/api/${api_version}/users" > "${fixture_output}" elif [ "${scenario}" = "api-cluster-admin-query" ] then get -H "X-Grafana-User: ceems-int-svc" "127.0.0.1:${port}/api/${api_version}/clusters/admin" > "${fixture_output}" @@ -913,6 +940,9 @@ then elif [ "${scenario}" = "api-running-query" ] then get -H "X-Grafana-User: test-user-1" "127.0.0.1:${port}/api/${api_version}/units?running&cluster_id=os-1&field=uuid&field=state&field=started_at&field=allocation&field=tags&timezone=${timezone}" > "${fixture_output}" + elif [ "${scenario}" = "api-units-query-k8s" ] + then + get -H "X-Grafana-User: kusr2" "127.0.0.1:${port}/api/${api_version}/units?cluster_id=k8s-1&project=ns2&to=1751883060" > "${fixture_output}" elif [ "${scenario}" = "api-admin-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/units/admin?user=usr3&cluster_id=slurm-0&project=acc3&from=1676934000&to=1677538800" > "${fixture_output}" @@ -931,27 +961,33 @@ then elif [ "${scenario}" = "api-current-usage-experimental-query" ] then get -H "X-Grafana-User: test-user-4" "127.0.0.1:${port}/api/${api_version}/usage/current?cluster_id=os-1&from=${usage_from}&to=${usage_to}&experimental" > "${fixture_output}" - elif [ "${scenario}" = "api-global-usage-query" ] - then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/usage/global?cluster_id=slurm-0&field=username&field=project&field=num_units" > "${fixture_output}" elif [ "${scenario}" = "api-current-usage-admin-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/usage/current/admin?cluster_id=slurm-1&user=usr15&user=usr3&from=${usage_from}&to=${usage_to}&__terminated" > "${fixture_output}" elif [ "${scenario}" = "api-current-usage-admin-experimental-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/usage/current/admin?cluster_id=slurm-1&user=usr15&user=usr4&cluster_id=os-1&user=test-user-4&from=${usage_from}&to=${usage_to}&experimental" > "${fixture_output}" + elif [ "${scenario}" = "api-current-usage-query-k8s" ] + then + get -H "X-Grafana-User: kusr2" "127.0.0.1:${port}/api/${api_version}/usage/current?cluster_id=k8s-1&from=${usage_from}&to=${usage_to}&__terminated" > "${fixture_output}" + elif [ "${scenario}" = "api-current-usage-admin-denied-query" ] + then + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/usage/global/admin?cluster_id=slurm-1&user=usr2" > "${fixture_output}" + elif [ "${scenario}" = "api-global-usage-query" ] + then + get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/usage/global?cluster_id=slurm-0&field=username&field=project&field=num_units" > "${fixture_output}" elif [ "${scenario}" = "api-global-usage-admin-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/usage/global/admin?cluster_id=slurm-0&field=username&field=project&field=num_units" > "${fixture_output}" + elif [ "${scenario}" = "api-global-usage-query-k8s" ] + then + get -H "X-Grafana-User: kusr1" "127.0.0.1:${port}/api/${api_version}/usage/global?cluster_id=slurm-0&field=username&field=project&field=num_units" > "${fixture_output}" elif [ "${scenario}" = "api-current-stats-admin-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/stats/current/admin?cluster_id=os-1&from=1728994800&to=1729005000" > "${fixture_output}" elif [ "${scenario}" = "api-global-stats-admin-query" ] then get -H "X-Grafana-User: grafana" "127.0.0.1:${port}/api/${api_version}/stats/global/admin" > "${fixture_output}" - elif [ "${scenario}" = "api-current-usage-admin-denied-query" ] - then - get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/usage/global/admin?cluster_id=slurm-1&user=usr2" > "${fixture_output}" elif [ "${scenario}" = "api-verify-pass-query" ] then get -H "X-Grafana-User: usr1" "127.0.0.1:${port}/api/${api_version}/units/verify?cluster_id=slurm-0&uuid=1479763&uuid=1479765" > "${fixture_output}" @@ -1115,13 +1151,14 @@ then elif [[ "${scenario}" = "lb-forbid-user-query-api" ]] then - ./bin/mock_servers prom pyro os-compute os-identity >> "${logfile}" 2>&1 & + ./bin/mock_servers prom pyro os-compute os-identity k8s-api >> "${logfile}" 2>&1 & MOCK_SERVERS_PID=$! waitport "9090" waitport "4040" waitport "8080" waitport "7070" + waitport "9080" # Copy config file to tmpdir cp pkg/api/testdata/config.yml "${tmpdir}/config.yml" @@ -1155,13 +1192,14 @@ then elif [[ "${scenario}" = "lb-allow-user-query-api" ]] then - ./bin/mock_servers prom pyro os-compute os-identity >> "${logfile}" 2>&1 & + ./bin/mock_servers prom pyro os-compute os-identity k8s-api >> "${logfile}" 2>&1 & MOCK_SERVERS_PID=$! waitport "4040" waitport "8080" waitport "7070" waitport "9090" + waitport "9080" # Copy config file to tmpdir cp pkg/api/testdata/config.yml "${tmpdir}/config.yml" @@ -1369,12 +1407,13 @@ then exit 1 fi - ./bin/mock_servers prom os-compute os-identity >> "${logfile}" 2>&1 & + ./bin/mock_servers prom os-compute os-identity k8s-api >> "${logfile}" 2>&1 & MOCK_SERVERS_PID=$! waitport "9090" waitport "8080" waitport "7070" + waitport "9080" # Copy config file to tmpdir cp pkg/api/testdata/config.yml "${tmpdir}/config.yml" diff --git a/scripts/mock_servers/main.go b/scripts/mock_servers/main.go index eaac48b6..759877ea 100644 --- a/scripts/mock_servers/main.go +++ b/scripts/mock_servers/main.go @@ -185,7 +185,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { uuidMatches := regexpUUID.FindAllStringSubmatch(query, -1) for _, match := range uuidMatches { if len(match) > 1 { - for _, uuid := range strings.Split(match[1], "|") { + for uuid := range strings.SplitSeq(match[1], "|") { // Ignore empty strings if strings.TrimSpace(uuid) != "" && !slices.Contains(uuids, uuid) { uuids = append(uuids, uuid) @@ -202,7 +202,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { // log.Println("Query", query, "UUIDs", uuids) - var results []interface{} + var results []any switch { case slices.Contains( @@ -218,12 +218,12 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { numDigits := lenLoop(h) value := float64(h) / math.Pow(10, float64(numDigits)-2) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "value": []interface{}{ + "value": []any{ 12345, strconv.FormatFloat(value, 'f', -1, 64), }, }) @@ -235,12 +235,12 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "value": []interface{}{ + "value": []any{ 12345, strconv.FormatUint(uint64(h), 10), }, }) @@ -252,12 +252,12 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "value": []interface{}{ + "value": []any{ 12345, strconv.FormatUint(uint64(h)*10, 10), }, }) @@ -269,12 +269,12 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "value": []interface{}{ + "value": []any{ 12345, strconv.FormatUint(uint64(h)*100, 10), }, }) @@ -286,12 +286,12 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "value": []interface{}{ + "value": []any{ 12345, strconv.FormatUint(uint64(h)*1000, 10), }, }) @@ -301,7 +301,7 @@ func QueryHandler(w http.ResponseWriter, r *http.Request) { // responseResults := filterResults(uuids, esults) response = tsdb.Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "resultType": "vector", "result": results, }, @@ -340,7 +340,7 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { uuidMatches := regexpUUID.FindAllStringSubmatch(query, -1) for _, match := range uuidMatches { if len(match) > 1 { - for _, uuid := range strings.Split(match[1], "|") { + for uuid := range strings.SplitSeq(match[1], "|") { // Ignore empty strings if strings.TrimSpace(uuid) != "" && !slices.Contains(uuids, uuid) { uuids = append(uuids, uuid) @@ -357,7 +357,7 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { // log.Println("Query", query, "UUIDs", uuids) - var results []interface{} + var results []any status := "success" @@ -374,16 +374,16 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { numDigits := lenLoop(h) value := float64(h) / math.Pow(10, float64(numDigits)-2) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{12345, strconv.FormatFloat(value, 'f', -1, 64)}, - []interface{}{12355, strconv.FormatFloat(value, 'f', -1, 64)}, - []interface{}{12365, strconv.FormatFloat(value, 'f', -1, 64)}, - []interface{}{12375, strconv.FormatFloat(value, 'f', -1, 64)}, + "values": []any{ + []any{12345, strconv.FormatFloat(value, 'f', -1, 64)}, + []any{12355, strconv.FormatFloat(value, 'f', -1, 64)}, + []any{12365, strconv.FormatFloat(value, 'f', -1, 64)}, + []any{12375, strconv.FormatFloat(value, 'f', -1, 64)}, }, }) } @@ -394,16 +394,16 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{12345, strconv.FormatUint(uint64(h), 10)}, - []interface{}{12355, strconv.FormatUint(uint64(h), 10)}, - []interface{}{12365, strconv.FormatUint(uint64(h), 10)}, - []interface{}{12375, strconv.FormatUint(uint64(h), 10)}, + "values": []any{ + []any{12345, strconv.FormatUint(uint64(h), 10)}, + []any{12355, strconv.FormatUint(uint64(h), 10)}, + []any{12365, strconv.FormatUint(uint64(h), 10)}, + []any{12375, strconv.FormatUint(uint64(h), 10)}, }, }) } @@ -414,16 +414,16 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{12345, strconv.FormatUint(uint64(h)*10, 10)}, - []interface{}{12355, strconv.FormatUint(uint64(h)*10, 10)}, - []interface{}{12365, strconv.FormatUint(uint64(h)*10, 10)}, - []interface{}{12375, strconv.FormatUint(uint64(h)*10, 10)}, + "values": []any{ + []any{12345, strconv.FormatUint(uint64(h)*10, 10)}, + []any{12355, strconv.FormatUint(uint64(h)*10, 10)}, + []any{12365, strconv.FormatUint(uint64(h)*10, 10)}, + []any{12375, strconv.FormatUint(uint64(h)*10, 10)}, }, }) } @@ -434,16 +434,16 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{12345, strconv.FormatUint(uint64(h)*100, 10)}, - []interface{}{12355, strconv.FormatUint(uint64(h)*100, 10)}, - []interface{}{12365, strconv.FormatUint(uint64(h)*100, 10)}, - []interface{}{12375, strconv.FormatUint(uint64(h)*100, 10)}, + "values": []any{ + []any{12345, strconv.FormatUint(uint64(h)*100, 10)}, + []any{12355, strconv.FormatUint(uint64(h)*100, 10)}, + []any{12365, strconv.FormatUint(uint64(h)*100, 10)}, + []any{12375, strconv.FormatUint(uint64(h)*100, 10)}, }, }) } @@ -454,16 +454,16 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { for _, uuid := range uuids { h := hash(uuid) results = append(results, - map[string]interface{}{ + map[string]any{ "metric": map[string]string{ "uuid": uuid, "instance": "localhost:9090", }, - "values": []interface{}{ - []interface{}{12345, strconv.FormatUint(uint64(h)*1000, 10)}, - []interface{}{12355, strconv.FormatUint(uint64(h)*1000, 10)}, - []interface{}{12365, strconv.FormatUint(uint64(h)*1000, 10)}, - []interface{}{12375, strconv.FormatUint(uint64(h)*1000, 10)}, + "values": []any{ + []any{12345, strconv.FormatUint(uint64(h)*1000, 10)}, + []any{12355, strconv.FormatUint(uint64(h)*1000, 10)}, + []any{12365, strconv.FormatUint(uint64(h)*1000, 10)}, + []any{12375, strconv.FormatUint(uint64(h)*1000, 10)}, }, }) } @@ -473,7 +473,7 @@ func QueryRangeHandler(w http.ResponseWriter, r *http.Request) { // responseResults := filterResults(uuids, esults) response = tsdb.Response[any]{ Status: status, - Data: map[string]interface{}{ + Data: map[string]any{ "resultType": "matrix", "result": results, }, @@ -500,7 +500,7 @@ func ConfigHandler(w http.ResponseWriter, r *http.Request) { func FlagsHandler(w http.ResponseWriter, r *http.Request) { response := tsdb.Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "query.lookback-delta": "5m", "query.max-samples": "50000000", "query.timeout": "2m", @@ -515,7 +515,7 @@ func FlagsHandler(w http.ResponseWriter, r *http.Request) { func RuntimeInfoHandler(w http.ResponseWriter, r *http.Request) { response := tsdb.Response[any]{ Status: "success", - Data: map[string]interface{}{ + Data: map[string]any{ "startTime": "2025-02-18T07:43:52.775090028Z", "CWD": "/var/lib/prometheus", "reloadConfigSuccess": true, @@ -568,7 +568,7 @@ func ServersHandler(w http.ResponseWriter, r *http.Request) { func TokensHandler(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var t map[string]interface{} + var t map[string]any if err := decoder.Decode(&t); err != nil { w.Write([]byte("KO")) @@ -619,6 +619,20 @@ func PodsListHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte("KO")) } +// RoleBindingsListHandler handles k8s rolebindings. +func RoleBindingsListHandler(w http.ResponseWriter, r *http.Request) { + if data, err := assetsFS.ReadFile("assets/k8s/rolebindings.json"); err == nil { + w.Header().Add("Content-Type", "application/json") + w.Header().Add("Content-Type", "application/vnd.kubernetes.protobuf") + w.Write(data) + + return + } + + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("KO")) +} + func redfishProxyTarget(ctx context.Context, i, portNum int, tls bool) { log.Println("Starting fake redfish target ", i, " on port", portNum) @@ -818,6 +832,7 @@ func k8sAPIServer(ctx context.Context) { // Registering our handler functions, and creating paths. k8sAPIMux := http.NewServeMux() k8sAPIMux.HandleFunc("/api/v1/pods", PodsListHandler) + k8sAPIMux.HandleFunc("/apis/rbac.authorization.k8s.io/v1/rolebindings", RoleBindingsListHandler) log.Println("Started k8s API server on port", k8sPortNum) log.Println("To close connection CTRL+C :-)") diff --git a/website/cspell.json b/website/cspell.json index 342451b2..ea6ff9eb 100644 --- a/website/cspell.json +++ b/website/cspell.json @@ -77,7 +77,8 @@ "cgroupfs", "CNRS", "OCC", - "kubeconfig" + "kubeconfig", + "kubeflow" ], // flagWords - list of words to be always considered incorrect // This is useful for offensive words and common spelling errors. diff --git a/website/docs/00-introduction.md b/website/docs/00-introduction.md index 84101f02..08833a56 100644 --- a/website/docs/00-introduction.md +++ b/website/docs/00-introduction.md @@ -32,7 +32,7 @@ managers (SLURM, Openstack, k8s) - Supports zero instrumentation eBPF based continuous profiling using [Grafana Pyroscope](https://grafana.com/oss/pyroscope/) as backend - Realtime access to metrics _via_ Grafana dashboards or a simple CLI tool -- Access control to Prometheus and Pyroscope datasources in Grafana +- Multi-tenancy and access control to Prometheus and Pyroscope datasources in Grafana - Stores aggregated metrics in a separate DB that can be retained for long time - CEEMS apps are [capability aware](https://tbhaxor.com/understanding-linux-capabilities/) diff --git a/website/docs/cli/ceems-api-server.md b/website/docs/cli/ceems-api-server.md index 1af3ba7c..6ea93e79 100644 --- a/website/docs/cli/ceems-api-server.md +++ b/website/docs/cli/ceems-api-server.md @@ -12,14 +12,14 @@ sidebar_position: 2 | `--version` | | Show application version. | | | `--log.format` | | Output format of log messages. One of: [logfmt, json] | `logfmt` | | `--log.level` | | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | +| `--config.file` | `CEEMS_API_SERVER_CONFIG_FILE` | Path to CEEMS API server configuration file | | +| `--config.file.expand-env-vars` | | Any environment variables that are referenced in config file will be expanded. To escape $ use $$ | `false` | | `--web.systemd-socket` | | Use systemd socket activation listeners instead of port listeners (Linux only). | `false` | -| `--runtime.gomaxprocs` | `GOMAXPROCS` | The target number of CPUs Go will run on | 1 | | `--web.config.file` | `CEEMS_API_SERVER_WEB_CONFIG_FILE` | Path to configuration file that can enable TLS or authentication. [Docs](https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md) | | | `--web.listen-address` | | Addresses on which to expose API server and web interface. | `:9020` | -| `--config.file` | `CEEMS_API_SERVER_CONFIG_FILE` | Path to CEEMS API server configuration file | | -| `--config.file.expand-env-vars` | | Any environment variables that are referenced in config file will be expanded. To escape $ use $$ | `false` | | `--web.route-prefix` | | Prefix for the internal routes of web endpoints | `/` | | `--web.max-requests` | | Maximum number of requests allowed in 1 minute period per client identified by Real IP address. Request headers `True-Client-IP`, `X-Real-IP` and `X-Forwarded-For` are looked up to get the real client IP address.By default no limit is applied | `0` | | `--web.cors.origin` | | Regex for CORS origin. It is fully anchored. Example: `https?://(domain1\|domain2)\.com` | `(.*)` | | `--web.debug-server` | | Enable /debug/pprof profiling endpoints | `false` | | `--query.max-period` | | Maximum allowable query range. Units Supported: y, w, d, h, m, s, ms. By default no limit is applied. | `0s` | +| `--runtime.gomaxprocs` | `GOMAXPROCS` | The target number of CPUs Go will run on | 1 | diff --git a/website/docs/components/ceems-api-server.md b/website/docs/components/ceems-api-server.md index 63d5fff3..5febf260 100644 --- a/website/docs/components/ceems-api-server.md +++ b/website/docs/components/ceems-api-server.md @@ -84,8 +84,7 @@ this translates to implementing two interfaces, one for fetching compute units a for fetching users and projects/namespaces/tenants data from the underlying resource manager. -Currently, the CEEMS API server includes SLURM support, and soon OpenStack support -will be added. +Currently, the CEEMS API server supports SLURM, Openstack and k8s resource managers. ### Updaters diff --git a/website/docs/configuration/ceems-api-server.md b/website/docs/configuration/ceems-api-server.md index 255eceef..ff011fa6 100644 --- a/website/docs/configuration/ceems-api-server.md +++ b/website/docs/configuration/ceems-api-server.md @@ -69,7 +69,7 @@ by `data.backup_interval` to a fault-tolerant storage. :::warning[WARNING] -As the database grows in size, the time taken to create a backup increases non-linearly +As the database grows in size, the time taken to create a backup increases exponentially and hence, use the backup option only if it is absolutely necessary. A general recommendation is to use a continuous backup solution like [litestream](https://litestream.io/) instead of the native backup solution offered @@ -131,6 +131,19 @@ clusters: user: name: admin password: supersecret + + - id: k8s-0 + manager: k8s + updaters: + - tsdb-0 + extra_config: + username_annotations: + - myk8s.io/created-by + - ceems.io/created-by + gpu_resource_names: + - nvidia.com/gpu + - amd.com/gpu + users_db_file: /var/run/ceems/users.yaml ``` Essentially, it is a list of objects where each object describes a cluster. @@ -138,7 +151,7 @@ Essentially, it is a list of objects where each object describes a cluster. - `id`: A unique identifier for each cluster. The identifier must stay consistent across CEEMS components, especially for CEEMS LB. More details can be found in the [Configuring CEEMS LB](./ceems-lb.md) section. -- `manager`: Resource manager kind. Currently, only `slurm` and `openstack` are +- `manager`: Resource manager kind. Currently, `slurm`, `openstack` and `k8s` are supported. - `updaters`: List of updaters to be used to update the aggregate metrics of compute units. The order is important as compute units are updated in the same order @@ -156,8 +169,8 @@ the OpenStack cluster's authentication is configured using the `web.http_headers All available options for the `web` configuration can be found in the [Web Client Configuration Reference](./config-reference.md#web_client_config). - `extra_config`: Any extra configuration required by a particular resource manager can be -provided here. Currently, the OpenStack resource manager uses this section to configure the API -URLs for compute and identity servers to fetch compute units, users and projects data. +provided here. Currently, the OpenStack and k8s resource managers uses this section to configure +extra information about API servers, users and projects. ### SLURM specific clusters configuration @@ -299,6 +312,88 @@ clusters: password: supersecret ``` +### k8s specific clusters configuration + +:::important[IMPORTANT] + +Before going into the k8s configuration, we strongly recommend users to read +[Kubernetes Section](./resource-managers.md#kubernetes) in +Resource Managers docs. + +::: + +In the case of k8s, the `extra_config` section resembles as follows: + +```yaml +extra_config: + kubeconfig_file: /path/to/out-of-cluster/kubeconfig.yaml + gpu_resource_names: + - nvidia.com/gpu + - amd.com/gpu + username_annotations: + - ceems.io/created-by + - exmaple.com/username + project_annotations: + - example.com/project + ns_users_list_file: /path/to/users.yaml +``` + +where each key is explained below: + +- `kubeconfig_file`: Path to the out-of-cluster kube config file to connect to +k8s API. In the k8s context, CEEMS API server app will be deployed as a pod and +hence, CEEMS API server will use in-cluster config. If in a scenario where CEEMS +API server is deployed outside of k8s cluster, this key can be used to configure +the kube config file of the target k8s cluster. +- `gpu_resource_names`: If the k8s cluster supports GPUs, this key take a list +of GPU resource names. By default it uses `[nvidia.com/gpu, amd.com/gpu]`. If +there any special resource names like NVIDIA MIG profiles, they must be explicitly +configured using this parameter. +- `username_annotations`: List of annotations names where username of the pod can be +retrieved. The first non empty string found in this list of annotations will be used +as the username. +- `project_annotations`: Similar to `username_annotations` but to retrieve the name +of the project. If not configured or none of the annotations exist on the pod, namespace +will be used as project in CEEMS DB. +- `ns_users_list_file`: When k8s cluster do not use native RBAC model to separate users +and projects, use this file to define a list of namespaces and the users that have +access to those namespaces. Effectively the user and namespace association described +in this file will be used in CEEMS DB to account for usage statistics and access control. + +A sample `ns_users_list_file` is shown below: + +```yaml +users: + ns1: + - usr1 + - usr2 + ns2: + - usr2 + - usr3 +``` + +Everytime a new namespace or user is created, this file must be updated so that CEEMS +API server will update its own DB with new associations. + +:::note[NOTE] + +CEEMS API server pulls users and their associated namespaces from rolebindings when the +cluster uses native RBAC model. In the cases where users and namespaces do not use native +RBAC model, use `ns_users_list_file` to setup users and namespaces + +::: + +#### Access control + +In the cases where end users interact with k8s clusters _via_ services like Argo CD, Kubeflow, +it is not possible to get the real user who is created k8s resources like pods, deployments, _etc_. +In such a scenario, it is not possible to impose "strict" access control by the CEEMS components +as pod ownership cannot be established reliably. In this case, using `ns_users_list_file` to +define users and their namespace associations can mitigate this issue to a certain extent. However, +even with `ns_users_list_file` defined and maintained, usage statistics and access control can only +be imposed at the project/namespace level and not at the user level. All the pods that have been +created by k8s service accounts will be made available to all users in a given project. + ## Updaters Configuration A sample updater config is shown below: diff --git a/website/docs/configuration/config-reference.md b/website/docs/configuration/config-reference.md index 1d7f175a..67ec2b4a 100644 --- a/website/docs/configuration/config-reference.md +++ b/website/docs/configuration/config-reference.md @@ -44,7 +44,7 @@ Generic placeholders are defined as follows: * ``: a regular string * ``: a size in bytes, e.g., `512MB`. A unit is required. Supported units: B, KB, MB, GB, TB, PB, EB. * ``: a string matching the regular expression `[a-zA-Z_-][a-zA-Z0-9_-]*`. Any other unsupported character in the source label should be converted to an underscore -* ``: a string that identifies the resource manager. Currently accepted values are `slurm`. +* ``: a string that identifies the resource manager. Currently accepted values are `slurm`, `openstack` and `k8s`. * ``: a string that identifies the updater type. Currently accepted values are `tsdb`. * ``: a valid PromQL query string. * ``: a valid load balancing strategy. Currently accepted values are `round-robin` and `least-connection`. @@ -283,8 +283,10 @@ A `cluster_config` allows configuring the cluster of the CEEMS API server. # id: -# Resource manager of the cluster. Currently, only `slurm` is supported. In the future, -# `openstack` will be supported. +# Resource manager of the cluster. Currently supported managers: +# - `slurm` +# - `openstack` +# - `k8s` # manager: @@ -362,7 +364,7 @@ web: # Any other configuration needed to reach the API server of the resource manager # can be configured in this section. # -# Currently, this section is used for the OpenStack resource manager +# Currently, this section is used for the OpenStack and k8s resource managers # to configure API versions. # # In the case of OpenStack, this section must have two keys: `api_service_endpoints` @@ -386,6 +388,58 @@ web: # name: admin # password: supersecret # +# In the case of k8s, this section is used to configure the k8s API related config +# +# Path to the kube config file when out-of-cluster config file is used. If the +# ceems_api_server is running in a pod, in-cluster config will be used by default. +# +# kubeconfig_file: '' +# +# List of annotation names where the name of the user that created/modified the pod is stored. +# The search for username will be stopped when the first annotation is found in the list. +# Therefore the order of the list is important. +# +# Default: [ceems.io/created-by] +# +# username_annotations: [] +# +# List of annotation names where the name of the project that the pod belongs to is stored. +# By default the project is always set to the namespace of the pod and when atleast one of +# annotation names are found in the pod spec, the namespace will be overridded by the value +# of the annotation +# +# project_annotations: [] +# +# List of GPU resource names in the cluster. +# +# Default: [nvidia.com/gpu, amd.com/gpu] +# +# gpu_resource_names: [] +# +# Path to the file that contains the list of namespaces and allowed users in each namespace. +# The format of the file must be as follows: +# +# users: +# ns1: +# - usr1 +# - usr2 +# ns2: +# - usr1 +# - usr3 +# +# ns_users_list_file: '' +# +# Example: +# +# extra_config: +# username_annotations: +# - ceems.io/created-by +# - example.io/created-by +# gpu_resource_names: +# - nvidia.com/gpu +# - nvidia.com/mig-1g.10gb +# ns_users_list_file: /var/run/ceems/users.yaml +# extra_config: [ : ... ] ``` @@ -522,12 +576,13 @@ A `queries_config` allows configuring PromQL queries for the TSDB updater of the # # The placeholder queries shown below should work out-of-the-box with the CEEMS # exporter when the recording rules to Prometheus have been configured -# using `ceems_tool`. If operators deploy more exporters of their own, queries +# using `ceems_tool`. Moreover the placeholders are the default values for different +# metrics. If operators deploy more exporters of their own, queries # must be modified accordingly. # # Average CPU utilization # -# Example of a valid query: +# Default value: # # global: # avg_over_time(avg by (uuid) (uuid:ceems_cpu_usage:ratio_irate{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:]) @@ -537,7 +592,7 @@ avg_cpu_usage: # Average CPU Memory utilization # -# Example of a valid query: +# Default value: # # global: # avg_over_time(avg by (uuid) (uuid:ceems_cpu_memory_usage:ratio{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:]) @@ -547,7 +602,7 @@ avg_cpu_mem_usage: # Total CPU energy usage in kWh # -# Example of a valid query: +# Default value: # # total: # sum_over_time(sum by (uuid) (uuid:ceems_host_power_watts:pue{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 3.6e9 @@ -557,7 +612,8 @@ total_cpu_energy_usage_kwh: # Total CPU emissions in gms # -# Example of a valid query: +# Default value: +# # rte_total: | # sum_over_time(sum by (uuid) (uuid:ceems_host_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="rte"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3 # emaps_total: | @@ -570,7 +626,7 @@ total_cpu_emissions_gms: # Average GPU utilization # -# Example of a valid query: +# Default value: # # global: # avg_over_time(avg by (uuid) (uuid:ceems_gpu_usage:ratio{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:]) @@ -580,7 +636,7 @@ avg_gpu_usage: # Average GPU memory utilization # -# Example of a valid query: +# Default value: # # global: # avg_over_time(avg by (uuid) (uuid:ceems_gpu_memory_usage:ratio{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:]) @@ -590,7 +646,7 @@ avg_gpu_mem_usage: # Total GPU energy usage in kWh # -# Example of a valid query: +# Default value: # # total: # sum_over_time(sum by (uuid) (uuid:ceems_gpu_power_watts:pue{uuid=~"{{.UUIDs}}"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 3.6e9 @@ -600,7 +656,7 @@ total_gpu_energy_usage_kwh: # Total GPU emissions in gms # -# Example of a valid query: +# Default value: # # rte_total: | # sum_over_time(sum by (uuid) (uuid:ceems_gpu_emissions_g_s:pue{uuid=~"{{.UUIDs}}",provider="rte"} >= 0 < inf)[{{.Range}}:{{.ScrapeInterval}}]) * {{.ScrapeIntervalMilli}} / 1e3 @@ -614,25 +670,33 @@ total_gpu_emissions_gms: # Total IO write in GB stats # -# Example of a valid query: +# Default value: # # bytes_total: | # sum by (uuid) (increase(ceems_ebpf_write_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) +# requests_total: | +# sum by (uuid) (increase(ceems_ebpf_write_requests_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) +# errors_total: | +# sum by (uuid) (increase(ceems_ebpf_write_errors_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) total_io_write_stats: [ : ... ] # Total IO read in GB stats # -# Example of a valid query: +# Default value: # # bytes_total: | # sum by (uuid) (increase(ceems_ebpf_read_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) +# requests_total: | +# sum by (uuid) (increase(ceems_ebpf_read_requests_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) +# errors_total: | +# sum by (uuid) (increase(ceems_ebpf_read_errors_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) total_io_read_stats: [ : ... ] # Total ingress traffic stats # -# Example of a valid query: +# Default value: # # bytes_total: | # sum by (uuid) (increase(ceems_ebpf_ingress_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) @@ -641,6 +705,8 @@ total_ingress_stats: # Total egress traffic stats # +# Default value: +# # bytes_total: | # sum by (uuid) (increase(ceems_ebpf_egress_bytes_total{uuid=~"{{.UUIDs}}"}[{{.Range}}]) >= 0 < inf) total_egress_stats: diff --git a/website/docs/configuration/grafana.md b/website/docs/configuration/grafana.md index 053340ce..96af9e5a 100644 --- a/website/docs/configuration/grafana.md +++ b/website/docs/configuration/grafana.md @@ -4,22 +4,29 @@ sidebar_position: 7 # Grafana -This document outlines the necessary configuration for the Grafana server and its datasources. +This document outlines the necessary configuration for the Grafana server +and its datasources. ## Grafana Server -The only configuration that needs to be added to the Grafana server in the `/etc/grafana/grafana.ini` file is the following: +The only configuration that needs to be added to the Grafana server in the +`/etc/grafana/grafana.ini` file is the following: ```ini [dataproxy] send_user_header = true ``` -Alternatively, the same can be done by setting the `GF_DATAPROXY_SEND_USER_HEADER=true` environment variable on the Grafana server. +Alternatively, the same can be done by setting the `GF_DATAPROXY_SEND_USER_HEADER=true` +environment variable on the Grafana server. ## Grafana Datasources -When using the CEEMS LB to provide access control and load balancing for TSDB servers, the Prometheus datasource in Grafana must be configured slightly differently than when using a regular native Prometheus server. As discussed in [CEEMS LB Configuration](./ceems-lb.md#matching-backendsid-with-clustersid), a custom header must be added to the queries sent to the CEEMS LB server. +When using the CEEMS LB to provide access control and load balancing for +TSDB servers, the Prometheus datasource in Grafana must be configured slightly +differently than when using a regular native Prometheus server. As discussed in +[CEEMS LB Configuration](./ceems-lb.md#matching-backendsid-with-clustersid), a +custom header must be added to the queries sent to the CEEMS LB server. For instance, if the CEEMS API server and CEEMS LB have the following configuration: @@ -81,7 +88,11 @@ ceems_lb: - http://tsdb-1-replica ``` -It is clear that there are two different clusters, `slurm-0` and `os-0`, and each cluster has its own TSDB server (`tsdb-0` and `tsdb-1`, respectively). In Grafana, a Prometheus datasource for each cluster must be configured to present the metrics of each cluster separately. Thus, the following provisioning configuration can be used to configure the datasources for each cluster: +It is clear that there are two different clusters, `slurm-0` and `os-0`, and each +cluster has its own TSDB server (`tsdb-0` and `tsdb-1`, respectively). In Grafana, +a Prometheus datasource for each cluster must be configured to present the metrics +of each cluster separately. Thus, the following provisioning configuration can be +used to configure the datasources for each cluster: ```yaml datasources: @@ -114,10 +125,49 @@ datasources: httpHeaderValue1: os-0 ``` -Internally, the CEEMS LB checks the `X-Ceems-Cluster-Id` header and forwards the request to the correct backend group based on the provided cluster ID. This ensures that a single instance of CEEMS LB can be used to load balance across multiple clusters. +Internally, the CEEMS LB checks the `X-Ceems-Cluster-Id` header and forwards the +request to the correct backend group based on the provided cluster ID. This ensures +that a single instance of CEEMS LB can be used to load balance across multiple clusters. :::important[IMPORTANT] -Even if there is only one cluster and one TSDB instance, the datasource in Grafana must be configured with the custom header as explained above if you wish to use the CEEMS LB. This is the only way for the CEEMS LB to know which cluster to target. +Even if there is only one cluster and one TSDB instance, the datasource in Grafana must +be configured with the custom header as explained above if you wish to use the CEEMS LB. +This is the only way for the CEEMS LB to know which cluster to target. ::: + +Similarly, CEEMS API server must be added as +[Grafana Infinity Datasource](https://grafana.com/docs/plugins/yesoreyeram-infinity-datasource/latest/) +datasource to pull the list of compute units and their usage metrics. Following snippet shows +a basic configuration for the datasource: + +:::important[IMPORTANT] + +Header name `X-Ceems-Disable-User-Check` must always be set even when the value is set to `false` +just to ensure that end users cannot spoof the header value from the browser requests. + +::: + +```yaml +# List of datasources that CEEMS uses +datasources: + # CEEMS API server JSON datasource + - name: ceems-api + type: yesoreyeram-infinity-datasource + url: http://ceems-api:9020 + basicAuth: true + basicAuthUser: + jsonData: + auth_method: basicAuth + timeout: 120 + allowedHosts: + - http://ceems-api:9020 + httpHeaderName1: X-Grafana-User + secureJsonData: + basicAuthPassword: + # This will be replaced by username before passing to API server + # This feature is available only for yesoreyeram-infinity-datasource >= 3.x + # IMPORTANT: Need $$ to escape $ + httpHeaderValue1: $${__user.login} +``` diff --git a/website/docs/configuration/resource-managers.md b/website/docs/configuration/resource-managers.md index d1757cb1..8bbac49a 100644 --- a/website/docs/configuration/resource-managers.md +++ b/website/docs/configuration/resource-managers.md @@ -4,29 +4,54 @@ sidebar_position: 5 # Resource Managers -This section contains information on the configuration required by the resource managers supported by CEEMS. +This section contains information on the configuration required by +the resource managers supported by CEEMS. ## SLURM -The [SLURM collector](../components/ceems-exporter.md#slurm-collector) in the CEEMS exporter relies on the job accounting information (like CPU time and memory usage) in the cgroups that SLURM creates for each job to estimate the energy and emissions for a given job. However, depending on the cgroups version and SLURM configuration, this accounting information might not be available. The following section provides guidelines on how to configure SLURM to ensure that this accounting information is always available. +The [SLURM collector](../components/ceems-exporter.md#slurm-collector) +in the CEEMS exporter relies on the job accounting information +(like CPU time and memory usage) in the cgroups that SLURM creates for +each job to estimate the energy and emissions for a given job. However, +depending on the cgroups version and SLURM configuration, this accounting +information might not be available. The following section provides guidelines +on how to configure SLURM to ensure that this accounting information is +always available. -Starting from [SLURM 22.05](https://slurm.schedmd.com/archive/slurm-22.05.0/cgroups.html), SLURM supports both cgroups v1 and v2. When using cgroups v1, SLURM might not contain accounting information in the cgroups. +Starting from [SLURM 22.05](https://slurm.schedmd.com/archive/slurm-22.05.0/cgroups.html), +SLURM supports both cgroups v1 and v2. When using cgroups v1, +SLURM might not contain accounting information in the cgroups. ### cgroups v1 -The following configuration enables the necessary cgroups controllers and provides the accounting information for jobs when cgroups v1 is used. +The following configuration enables the necessary cgroups controllers +and provides the accounting information for jobs when cgroups v1 is used. -As stated in the [cgroups docs of SLURM](https://slurm.schedmd.com/cgroup.conf.html), the cgroups plugin can be controlled by the configuration in this file. An [example config](https://slurm.schedmd.com/cgroup.conf.html#OPT_/etc/slurm/cgroup.conf) is also provided, which serves as a good starting point. +As stated in the [cgroups docs of SLURM](https://slurm.schedmd.com/cgroup.conf.html), +the cgroups plugin can be controlled by the configuration in this file. +An [example config](https://slurm.schedmd.com/cgroup.conf.html#OPT_/etc/slurm/cgroup.conf) +is also provided, which serves as a good starting point. -Along with the `cgroups.conf` file, certain configuration parameters are required in the `slurm.conf` file as well. This information is provided in the [SLURM docs](https://slurm.schedmd.com/cgroup.conf.html#OPT_/etc/slurm/slurm.conf) as well. +Along with the `cgroups.conf` file, certain configuration parameters are +required in the `slurm.conf` file as well. This information is provided +in the [SLURM docs](https://slurm.schedmd.com/cgroup.conf.html#OPT_/etc/slurm/slurm.conf) +as well. :::important[IMPORTANT] -Although `JobAcctGatherType=jobacct_gather/cgroup` is presented as an _optional_ configuration parameter, it _must_ be used to get the accounting information for CPU usage. Without this configuration parameter, the CPU time of the job will not be available in the job's cgroups. +Although `JobAcctGatherType=jobacct_gather/cgroup` is presented as an +_optional_ configuration parameter, it _must_ be used to get the accounting +information for CPU usage. Without this configuration parameter, the CPU +time of the job will not be available in the job's cgroups. ::: -Besides the above configuration, [SelectTypeParameters](https://slurm.schedmd.com/slurm.conf.html#OPT_SelectTypeParameters) must be configured to set the core or CPU and memory as consumable resources. This is highlighted in the documentation of the [ConstrainRAMSpace](https://slurm.schedmd.com/cgroup.conf.html#OPT_ConstrainRAMSpace) configuration parameter in the [`cgroups.conf` docs](https://slurm.schedmd.com/cgroup.conf.html). +Besides the above configuration, +[SelectTypeParameters](https://slurm.schedmd.com/slurm.conf.html#OPT_SelectTypeParameters) +must be configured to set the core or CPU and memory as consumable resources. +This is highlighted in the documentation of the +[ConstrainRAMSpace](https://slurm.schedmd.com/cgroup.conf.html#OPT_ConstrainRAMSpace) +configuration parameter in the [`cgroups.conf` docs](https://slurm.schedmd.com/cgroup.conf.html). In conclusion, here are the necessary configuration excerpts: @@ -52,8 +77,123 @@ AccountingStorageTRES=gres/gpu # or any other TRES resources declared in your SL ### cgroups v2 -For cgroups v2, SLURM should create the proper cgroups for every job without any special configuration. However, the configuration presented for [cgroups v1](#cgroups-v1) is applicable to cgroups v2, and it is advised to use that configuration for cgroups v2 as well. +For cgroups v2, SLURM should create the proper cgroups for every job without +any special configuration. However, the configuration presented for +[cgroups v1](#cgroups-v1) is applicable to cgroups v2, and it is advised to use +that configuration for cgroups v2 as well. ## Libvirt -The libvirt collector is meant to be used for OpenStack clusters. There is no special configuration needed, as OpenStack will take care of configuring libvirt and QEMU to enable all relevant cgroup controllers. +The libvirt collector is meant to be used for OpenStack clusters. There is no +special configuration needed, as OpenStack will take care of configuring libvirt +and QEMU to enable all relevant cgroup controllers. + +## Kubernetes + +Unlike SLURM and Openstack, Kubernetes do not have a concept of user object. Any user +with a valid kubeconfig file can create k8s resources like Pod, Deployment, _etc_. Thus, +it is not possible to attribute a pod to a user natively in Kubernetes. Moreover, pod +spec only contains the namespace that the pod is created in and not the name of the +user who created that pod resource. This poses a significant challenge for CEEMS as +a definite pod to user association is needed to impose estimate the user's usage +statistics and strict access control. + +### Admission Controller + +In order to address above mentioned limitation of Kubernetes, CEEMS ships an admission +controller to add the name of the user to the resource spec as an annotation. By default, +the admission controller will add the name of the user who is creating that resource as +an annotation `ceems.io/created-by` to that resource. This annotation is eventually passed +down to the pod which is the most primitive unit of compute in Kubernetes context. For +instance, if a user creates a Deployment, the admission controller of the CEEMS will add +an annotation `ceems.io/created-by` with username as the value. However, it is the Deployment +controller service account which creates the pods defined in the Deployment spec. Instead +of adding the name of that controller service account, the admission controller ensures the +name of the user in `ceems.io/created-by` is passed down to the pod as well. Eventually, +CEEMS API server can be configured to search for a list of annotations to get the username +from. + +:::important[IMPORTANT] + +The admission controller shipped by CEEMS do not stop anyone from creating Kubernetes +resources. It only adds annotations to the resource specs in a mutation hook and +checks for the presence of the required annotation in validate hook. Even if the +required annotation is absent in the resource spec, it passes the hook with a log +entry for debugging purposes for the operators. + +::: + +For this approach to work, there should be a strict RBAC model that must be followed. +Namespaces must be used as abstraction for different projects and users must be confined +to different namespaces. Moreover, this needs that end users should create the Kubernetes +resources themselves instead of some other service for them. For example, let's take +examples of [JupyterHub](https://z2jh.jupyter.org/en/stable/) and +[Kubeflow](https://www.kubeflow.org/). In both these cases, end users creates Kubernetes +resources _via_ web UI interface where service accounts creates pod on behalf of users. +In these cases, Admission controller will add the name of the service account to the +annotation as retrieving the "real" username is not trivial. + +#### JupyterHub + +In the case of JupyterHub, there is a workaround using +[`extra_annotations`](https://jupyterhub-kubespawner.readthedocs.io/en/latest/spawner.html#kubespawner.KubeSpawner.extra_annotations) +configuration parameter supported by +[KubeSpawner](https://jupyterhub-kubespawner.readthedocs.io/en/latest/index.html). +By setting the following configuration parameter, each pod can be annotated with the +current username. + +```python +c.KubeSpawner.extra_annotations = {'ceems.io/created-by': f'{username}'} +``` + +For a better isolation, each project must use its own namespace and single user +servers must use the project's namespace. This can be achieved in different ways +and we recommend to consult the +[KubeSpawner's API docs](https://jupyterhub-kubespawner.readthedocs.io/en/latest/spawner.html#). + +#### Kubeflow + +Kubeflow uses Kubernetes RBAC model to support +[multi-tenancy](https://www.kubeflow.org/docs/components/central-dash/profiles/). +This means each project will have a dedicated namespace and users will have roles +created to create resources in their project namespaces. Thus, the association +between the project namespaces and users can be retrieved by listing all the rolebindings +of Kubernetes cluster. This is what CEEMS API server does to maintain a list of users +and their namespaces. However, there is no easy way to add the username annotation to +the Kubernetes resources created by Kubeflow. The consequence of this limitation is that +CEEMS can only estimate usage metrics at the project level and not really user level. +Operators can encourage the users to add their username as a custom annotation to their +resource specs and configure the name of that custom annotation with CEEMS API server +to enable accounting of user statistics. + +### CRDs + +The admission controller that is shipped with CEEMS only works for default Kubernetes +resources and not with CRDs. Thus usage accounting and access control of CEEMS cannot +support CRDs like [Argo CD](https://argo-cd.readthedocs.io/en/stable/), +[Argo Workflows](https://argoproj.github.io/workflows/). A workaround is available in +this kind of scenario as well. + +For example, depending on the cluster configuration, Argo CD allows to deploy +resources to arbitrary namespace. And Argo CD uses its own +[RBAC model](https://argo-cd.readthedocs.io/en/stable/operator-manual/rbac/) and +do not rely on Kubernetes RBAC model. Thus, it is not possible to get users and their +associated namespaces by simply listing rolebindings. The operators needs to maintain +this association in a different file and CEEMS API server is capable of reading the +usernames and their namespaces from this file. This is a very simple YAML file containing +name of the namespace as key and list of usernames as values. For instance, the a sample +file can be as follows: + +```yaml +users: + ns1: + - usr1 + - usr2 + ns2: + - usr2 + - usr3 +``` + +Typically, in a Kubernetes cluster this information can be created as a configmap and +mounted on the CEEMS API server pod as a file. This allows CEEMS to account the usage +statistics per namespace and allow to impose access control. diff --git a/website/docs/usage/ceems-api-server.md b/website/docs/usage/ceems-api-server.md index 20f2eca0..c68505c2 100644 --- a/website/docs/usage/ceems-api-server.md +++ b/website/docs/usage/ceems-api-server.md @@ -14,6 +14,19 @@ as follows: ceems_api_server --config.file=/path/core/config/file --web.config.file=/path/to/web/config/file ``` +If there are environment variables used in `/path/core/config/file`, they can be expanded using + +```bash +ceems_api_server --config.file=/path/core/config/file --config.file.expand-env-vars --web.config.file=/path/to/web/config/file +``` + +:::tip[TIP] + +In order to escape `$` in the config file when CLI flag `--config.file.expand-env-vars` is used, +use `$$`. + +::: + The default interface and port can be changed using the `--web.listen-address` CLI argument: ```bash diff --git a/website/docs/usage/ceems-exporter.md b/website/docs/usage/ceems-exporter.md index 04b6445d..6fe00bbb 100644 --- a/website/docs/usage/ceems-exporter.md +++ b/website/docs/usage/ceems-exporter.md @@ -6,13 +6,6 @@ sidebar_position: 1 ## Basic Usage -:::important[IMPORTANT] - -Currently, the CEEMS exporter supports exporting SLURM job and OpenStack VM metrics. -Support for Kubernetes is planned for the next milestone. - -::: - To run the exporter with the default enabled collectors, use the following command: ```bash diff --git a/website/docs/usage/ceems-lb.md b/website/docs/usage/ceems-lb.md index 72bf2da9..517ba3eb 100644 --- a/website/docs/usage/ceems-lb.md +++ b/website/docs/usage/ceems-lb.md @@ -12,6 +12,19 @@ The CEEMS load balancer can be started with its core and web configuration files ceems_lb --config.file=/path/to/core/config/file --web.config.file=/path/to/web/config/file ``` +If there are environment variables used in `/path/core/config/file`, they can be expanded using + +```bash +ceems_lb --config.file=/path/core/config/file --config.file.expand-env-vars --web.config.file=/path/to/web/config/file +``` + +:::tip[TIP] + +In order to escape `$` in the config file when CLI flag `--config.file.expand-env-vars` is used, +use `$$`. + +::: + This will start the CEEMS load balancer at the default port `9030`, listening on all interfaces. To change the default port and host, the `--web.listen-address` CLI argument must be passed to the binary: