From 3ece501f6b416e12545bbd25b22fe63a84600713 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Mon, 12 Nov 2018 11:02:49 -0800 Subject: [PATCH 1/4] add kubelet pod-resources API definition --- .../apis/podresources/v1alpha1/api.proto | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 pkg/kubelet/apis/podresources/v1alpha1/api.proto diff --git a/pkg/kubelet/apis/podresources/v1alpha1/api.proto b/pkg/kubelet/apis/podresources/v1alpha1/api.proto new file mode 100644 index 0000000000000..eedcd59d73704 --- /dev/null +++ b/pkg/kubelet/apis/podresources/v1alpha1/api.proto @@ -0,0 +1,48 @@ +// To regenerate api.pb.go run hack/update-generated-pod-resources.sh +syntax = 'proto3'; + +package v1alpha1; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +option (gogoproto.goproto_stringer_all) = false; +option (gogoproto.stringer_all) = true; +option (gogoproto.goproto_getters_all) = true; +option (gogoproto.marshaler_all) = true; +option (gogoproto.sizer_all) = true; +option (gogoproto.unmarshaler_all) = true; +option (gogoproto.goproto_unrecognized_all) = false; + + +// PodResourcesLister is a service provided by the kubelet that provides information about the +// node resources consumed by pods and containers on the node +service PodResourcesLister { + rpc List(ListPodResourcesRequest) returns (ListPodResourcesResponse) {} +} + +// ListPodResourcesRequest is the request made to the PodResourcesLister service +message ListPodResourcesRequest {} + +// ListPodResourcesResponse is the response returned by List function +message ListPodResourcesResponse { + repeated PodResources pod_resources = 1; +} + +// PodResources contains information about the node resources assigned to a pod +message PodResources { + string name = 1; + string namespace = 2; + repeated ContainerResources containers = 3; +} + +// ContainerResources contains information about the resources assigned to a container +message ContainerResources { + string name = 1; + repeated ContainerDevices devices = 2; +} + +// ContainerDevices contains information about the devices assigned to a container +message ContainerDevices { + string resource_name = 1; + repeated string device_ids = 2; +} \ No newline at end of file From aa9ba976ac6e9445364130123e496e1313a4adc7 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Tue, 13 Nov 2018 18:00:24 -0800 Subject: [PATCH 2/4] generate go client for kubelet podresources API --- ...date-generated-pod-resources-dockerized.sh | 25 + hack/update-generated-pod-resources.sh | 27 + hack/verify-generated-pod-resources.sh | 45 + pkg/kubelet/apis/podresources/v1alpha1/BUILD | 28 + .../apis/podresources/v1alpha1/api.pb.go | 1182 +++++++++++++++++ 5 files changed, 1307 insertions(+) create mode 100755 hack/update-generated-pod-resources-dockerized.sh create mode 100755 hack/update-generated-pod-resources.sh create mode 100755 hack/verify-generated-pod-resources.sh create mode 100644 pkg/kubelet/apis/podresources/v1alpha1/BUILD create mode 100644 pkg/kubelet/apis/podresources/v1alpha1/api.pb.go diff --git a/hack/update-generated-pod-resources-dockerized.sh b/hack/update-generated-pod-resources-dockerized.sh new file mode 100755 index 0000000000000..d11b136b59f90 --- /dev/null +++ b/hack/update-generated-pod-resources-dockerized.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE}")/../" && pwd -P)" +POD_RESOURCES_ALPHA="${KUBE_ROOT}/pkg/kubelet/apis/podresources/v1alpha1/" + +source "${KUBE_ROOT}/hack/lib/protoc.sh" +kube::protoc::generate_proto ${POD_RESOURCES_ALPHA} diff --git a/hack/update-generated-pod-resources.sh b/hack/update-generated-pod-resources.sh new file mode 100755 index 0000000000000..bdf8c584c0e57 --- /dev/null +++ b/hack/update-generated-pod-resources.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +# NOTE: All output from this script needs to be copied back to the calling +# source tree. This is managed in kube::build::copy_output in build/common.sh. +# If the output set is changed update that function. + +${KUBE_ROOT}/build/run.sh hack/update-generated-pod-resources-dockerized.sh "$@" diff --git a/hack/verify-generated-pod-resources.sh b/hack/verify-generated-pod-resources.sh new file mode 100755 index 0000000000000..84213cc969807 --- /dev/null +++ b/hack/verify-generated-pod-resources.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. +KUBE_REMOTE_RUNTIME_ROOT="${KUBE_ROOT}/pkg/kubelet/apis/podresources/v1alpha1" +source "${KUBE_ROOT}/hack/lib/init.sh" + +kube::golang::setup_env + +function cleanup { + rm -rf ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/ +} + +trap cleanup EXIT + +mkdir -p ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp +cp ${KUBE_REMOTE_RUNTIME_ROOT}/api.pb.go ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/ + +ret=0 +KUBE_VERBOSE=3 "${KUBE_ROOT}/hack/update-generated-pod-resources.sh" +diff -I "gzipped FileDescriptorProto" -I "0x" -Naupr ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/api.pb.go ${KUBE_REMOTE_RUNTIME_ROOT}/api.pb.go || ret=$? +if [[ $ret -eq 0 ]]; then + echo "Generated pod resources api is up to date." + cp ${KUBE_REMOTE_RUNTIME_ROOT}/_tmp/api.pb.go ${KUBE_REMOTE_RUNTIME_ROOT}/ +else + echo "Generated pod resources api is out of date. Please run hack/update-generated-pod-resources.sh" + exit 1 +fi diff --git a/pkg/kubelet/apis/podresources/v1alpha1/BUILD b/pkg/kubelet/apis/podresources/v1alpha1/BUILD new file mode 100644 index 0000000000000..a56deb71467cc --- /dev/null +++ b/pkg/kubelet/apis/podresources/v1alpha1/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["api.pb.go"], + importpath = "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library", + "//vendor/github.com/gogo/protobuf/proto:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubelet/apis/podresources/v1alpha1/api.pb.go b/pkg/kubelet/apis/podresources/v1alpha1/api.pb.go new file mode 100644 index 0000000000000..df4d4300f21a2 --- /dev/null +++ b/pkg/kubelet/apis/podresources/v1alpha1/api.pb.go @@ -0,0 +1,1182 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: api.proto + +/* + Package v1alpha1 is a generated protocol buffer package. + + It is generated from these files: + api.proto + + It has these top-level messages: + ListPodResourcesRequest + ListPodResourcesResponse + PodResources + ContainerResources + ContainerDevices +*/ +package v1alpha1 + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "github.com/gogo/protobuf/gogoproto" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import strings "strings" +import reflect "reflect" + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +// ListPodResourcesRequest is the request made to the PodResourcesLister service +type ListPodResourcesRequest struct { +} + +func (m *ListPodResourcesRequest) Reset() { *m = ListPodResourcesRequest{} } +func (*ListPodResourcesRequest) ProtoMessage() {} +func (*ListPodResourcesRequest) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{0} } + +// ListPodResourcesResponse is the response returned by List function +type ListPodResourcesResponse struct { + PodResources []*PodResources `protobuf:"bytes,1,rep,name=pod_resources,json=podResources" json:"pod_resources,omitempty"` +} + +func (m *ListPodResourcesResponse) Reset() { *m = ListPodResourcesResponse{} } +func (*ListPodResourcesResponse) ProtoMessage() {} +func (*ListPodResourcesResponse) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{1} } + +func (m *ListPodResourcesResponse) GetPodResources() []*PodResources { + if m != nil { + return m.PodResources + } + return nil +} + +// PodResources contains information about the node resources assigned to a pod +type PodResources struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Namespace string `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` + Containers []*ContainerResources `protobuf:"bytes,3,rep,name=containers" json:"containers,omitempty"` +} + +func (m *PodResources) Reset() { *m = PodResources{} } +func (*PodResources) ProtoMessage() {} +func (*PodResources) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{2} } + +func (m *PodResources) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PodResources) GetNamespace() string { + if m != nil { + return m.Namespace + } + return "" +} + +func (m *PodResources) GetContainers() []*ContainerResources { + if m != nil { + return m.Containers + } + return nil +} + +// ContainerResources contains information about the resources assigned to a container +type ContainerResources struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Devices []*ContainerDevices `protobuf:"bytes,2,rep,name=devices" json:"devices,omitempty"` +} + +func (m *ContainerResources) Reset() { *m = ContainerResources{} } +func (*ContainerResources) ProtoMessage() {} +func (*ContainerResources) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{3} } + +func (m *ContainerResources) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ContainerResources) GetDevices() []*ContainerDevices { + if m != nil { + return m.Devices + } + return nil +} + +// ContainerDevices contains information about the devices assigned to a container +type ContainerDevices struct { + ResourceName string `protobuf:"bytes,1,opt,name=resource_name,json=resourceName,proto3" json:"resource_name,omitempty"` + DeviceIds []string `protobuf:"bytes,2,rep,name=device_ids,json=deviceIds" json:"device_ids,omitempty"` +} + +func (m *ContainerDevices) Reset() { *m = ContainerDevices{} } +func (*ContainerDevices) ProtoMessage() {} +func (*ContainerDevices) Descriptor() ([]byte, []int) { return fileDescriptorApi, []int{4} } + +func (m *ContainerDevices) GetResourceName() string { + if m != nil { + return m.ResourceName + } + return "" +} + +func (m *ContainerDevices) GetDeviceIds() []string { + if m != nil { + return m.DeviceIds + } + return nil +} + +func init() { + proto.RegisterType((*ListPodResourcesRequest)(nil), "v1alpha1.ListPodResourcesRequest") + proto.RegisterType((*ListPodResourcesResponse)(nil), "v1alpha1.ListPodResourcesResponse") + proto.RegisterType((*PodResources)(nil), "v1alpha1.PodResources") + proto.RegisterType((*ContainerResources)(nil), "v1alpha1.ContainerResources") + proto.RegisterType((*ContainerDevices)(nil), "v1alpha1.ContainerDevices") +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for PodResourcesLister service + +type PodResourcesListerClient interface { + List(ctx context.Context, in *ListPodResourcesRequest, opts ...grpc.CallOption) (*ListPodResourcesResponse, error) +} + +type podResourcesListerClient struct { + cc *grpc.ClientConn +} + +func NewPodResourcesListerClient(cc *grpc.ClientConn) PodResourcesListerClient { + return &podResourcesListerClient{cc} +} + +func (c *podResourcesListerClient) List(ctx context.Context, in *ListPodResourcesRequest, opts ...grpc.CallOption) (*ListPodResourcesResponse, error) { + out := new(ListPodResourcesResponse) + err := grpc.Invoke(ctx, "/v1alpha1.PodResourcesLister/List", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for PodResourcesLister service + +type PodResourcesListerServer interface { + List(context.Context, *ListPodResourcesRequest) (*ListPodResourcesResponse, error) +} + +func RegisterPodResourcesListerServer(s *grpc.Server, srv PodResourcesListerServer) { + s.RegisterService(&_PodResourcesLister_serviceDesc, srv) +} + +func _PodResourcesLister_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListPodResourcesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PodResourcesListerServer).List(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/v1alpha1.PodResourcesLister/List", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PodResourcesListerServer).List(ctx, req.(*ListPodResourcesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _PodResourcesLister_serviceDesc = grpc.ServiceDesc{ + ServiceName: "v1alpha1.PodResourcesLister", + HandlerType: (*PodResourcesListerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "List", + Handler: _PodResourcesLister_List_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "api.proto", +} + +func (m *ListPodResourcesRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ListPodResourcesRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func (m *ListPodResourcesResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ListPodResourcesResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.PodResources) > 0 { + for _, msg := range m.PodResources { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *PodResources) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PodResources) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Namespace) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Namespace))) + i += copy(dAtA[i:], m.Namespace) + } + if len(m.Containers) > 0 { + for _, msg := range m.Containers { + dAtA[i] = 0x1a + i++ + i = encodeVarintApi(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *ContainerResources) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerResources) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Name) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.Name))) + i += copy(dAtA[i:], m.Name) + } + if len(m.Devices) > 0 { + for _, msg := range m.Devices { + dAtA[i] = 0x12 + i++ + i = encodeVarintApi(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *ContainerDevices) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ContainerDevices) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.ResourceName) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintApi(dAtA, i, uint64(len(m.ResourceName))) + i += copy(dAtA[i:], m.ResourceName) + } + if len(m.DeviceIds) > 0 { + for _, s := range m.DeviceIds { + dAtA[i] = 0x12 + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + return i, nil +} + +func encodeVarintApi(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *ListPodResourcesRequest) Size() (n int) { + var l int + _ = l + return n +} + +func (m *ListPodResourcesResponse) Size() (n int) { + var l int + _ = l + if len(m.PodResources) > 0 { + for _, e := range m.PodResources { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *PodResources) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + l = len(m.Namespace) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.Containers) > 0 { + for _, e := range m.Containers { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *ContainerResources) Size() (n int) { + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.Devices) > 0 { + for _, e := range m.Devices { + l = e.Size() + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func (m *ContainerDevices) Size() (n int) { + var l int + _ = l + l = len(m.ResourceName) + if l > 0 { + n += 1 + l + sovApi(uint64(l)) + } + if len(m.DeviceIds) > 0 { + for _, s := range m.DeviceIds { + l = len(s) + n += 1 + l + sovApi(uint64(l)) + } + } + return n +} + +func sovApi(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozApi(x uint64) (n int) { + return sovApi(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *ListPodResourcesRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ListPodResourcesRequest{`, + `}`, + }, "") + return s +} +func (this *ListPodResourcesResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ListPodResourcesResponse{`, + `PodResources:` + strings.Replace(fmt.Sprintf("%v", this.PodResources), "PodResources", "PodResources", 1) + `,`, + `}`, + }, "") + return s +} +func (this *PodResources) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&PodResources{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, + `Containers:` + strings.Replace(fmt.Sprintf("%v", this.Containers), "ContainerResources", "ContainerResources", 1) + `,`, + `}`, + }, "") + return s +} +func (this *ContainerResources) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ContainerResources{`, + `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Devices:` + strings.Replace(fmt.Sprintf("%v", this.Devices), "ContainerDevices", "ContainerDevices", 1) + `,`, + `}`, + }, "") + return s +} +func (this *ContainerDevices) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ContainerDevices{`, + `ResourceName:` + fmt.Sprintf("%v", this.ResourceName) + `,`, + `DeviceIds:` + fmt.Sprintf("%v", this.DeviceIds) + `,`, + `}`, + }, "") + return s +} +func valueToStringApi(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *ListPodResourcesRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListPodResourcesRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListPodResourcesRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ListPodResourcesResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ListPodResourcesResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ListPodResourcesResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PodResources", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PodResources = append(m.PodResources, &PodResources{}) + if err := m.PodResources[len(m.PodResources)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PodResources) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PodResources: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PodResources: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Namespace = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Containers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Containers = append(m.Containers, &ContainerResources{}) + if err := m.Containers[len(m.Containers)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerResources) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerResources: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerResources: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Devices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Devices = append(m.Devices, &ContainerDevices{}) + if err := m.Devices[len(m.Devices)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ContainerDevices) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ContainerDevices: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ContainerDevices: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ResourceName", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ResourceName = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DeviceIds", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowApi + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthApi + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DeviceIds = append(m.DeviceIds, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipApi(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthApi + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipApi(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthApi + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowApi + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipApi(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthApi = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowApi = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("api.proto", fileDescriptorApi) } + +var fileDescriptorApi = []byte{ + // 343 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0xb1, 0x4e, 0xc3, 0x30, + 0x10, 0xad, 0xdb, 0x0a, 0xc8, 0xd1, 0x4a, 0xc8, 0x03, 0x84, 0xaa, 0x58, 0xc5, 0x2c, 0x5d, 0x48, + 0xd5, 0xc2, 0x06, 0x13, 0xb0, 0x20, 0x21, 0x40, 0x19, 0x60, 0xa3, 0x4a, 0x13, 0xd3, 0x46, 0xa2, + 0xb1, 0x89, 0x93, 0x8e, 0x88, 0x4f, 0xe0, 0xb3, 0x3a, 0x32, 0x32, 0xd2, 0xf0, 0x23, 0x28, 0xb6, + 0xac, 0x04, 0x5a, 0x98, 0x7c, 0x77, 0xef, 0x9d, 0xdf, 0xf3, 0x9d, 0xc1, 0xf2, 0x44, 0xe8, 0x88, + 0x98, 0x27, 0x1c, 0x6f, 0xcc, 0xfa, 0xde, 0x93, 0x98, 0x78, 0xfd, 0xd6, 0xe1, 0x38, 0x4c, 0x26, + 0xe9, 0xc8, 0xf1, 0xf9, 0xb4, 0x37, 0xe6, 0x63, 0xde, 0x53, 0x84, 0x51, 0xfa, 0xa8, 0x32, 0x95, + 0xa8, 0x48, 0x37, 0xd2, 0x5d, 0xd8, 0xb9, 0x0a, 0x65, 0x72, 0xcb, 0x03, 0x97, 0x49, 0x9e, 0xc6, + 0x3e, 0x93, 0x2e, 0x7b, 0x4e, 0x99, 0x4c, 0xe8, 0x3d, 0xd8, 0xcb, 0x90, 0x14, 0x3c, 0x92, 0x0c, + 0x9f, 0x40, 0x53, 0xf0, 0x60, 0x18, 0x1b, 0xc0, 0x46, 0x9d, 0x5a, 0x77, 0x73, 0xb0, 0xed, 0x18, + 0x1f, 0xce, 0x8f, 0xb6, 0x86, 0x28, 0x65, 0xf4, 0x05, 0x1a, 0x65, 0x14, 0x63, 0xa8, 0x47, 0xde, + 0x94, 0xd9, 0xa8, 0x83, 0xba, 0x96, 0xab, 0x62, 0xdc, 0x06, 0x2b, 0x3f, 0xa5, 0xf0, 0x7c, 0x66, + 0x57, 0x15, 0x50, 0x14, 0xf0, 0x29, 0x80, 0xcf, 0xa3, 0xc4, 0x0b, 0x23, 0x16, 0x4b, 0xbb, 0xa6, + 0xb4, 0xdb, 0x85, 0xf6, 0xb9, 0xc1, 0x0a, 0x07, 0x25, 0x3e, 0x7d, 0x00, 0xbc, 0xcc, 0x58, 0xe9, + 0xe2, 0x18, 0xd6, 0x03, 0x36, 0x0b, 0xf3, 0x07, 0x56, 0x95, 0x48, 0x6b, 0x85, 0xc8, 0x85, 0x66, + 0xb8, 0x86, 0x4a, 0xef, 0x60, 0xeb, 0x37, 0x88, 0x0f, 0xa0, 0x69, 0x86, 0x35, 0x2c, 0xc9, 0x34, + 0x4c, 0xf1, 0x3a, 0x97, 0xdb, 0x03, 0xd0, 0x77, 0x0c, 0xc3, 0x40, 0x2b, 0x5a, 0xae, 0xa5, 0x2b, + 0x97, 0x81, 0x1c, 0x30, 0xc0, 0xe5, 0xb9, 0xe5, 0xcb, 0x61, 0x31, 0xbe, 0x81, 0x7a, 0x1e, 0xe1, + 0xfd, 0xc2, 0xda, 0x1f, 0x1b, 0x6d, 0xd1, 0xff, 0x28, 0x7a, 0xb3, 0xb4, 0x72, 0xd6, 0x9e, 0x2f, + 0x08, 0xfa, 0x58, 0x90, 0xca, 0x6b, 0x46, 0xd0, 0x3c, 0x23, 0xe8, 0x3d, 0x23, 0xe8, 0x33, 0x23, + 0xe8, 0xed, 0x8b, 0x54, 0x46, 0x6b, 0xea, 0xdf, 0x1c, 0x7d, 0x07, 0x00, 0x00, 0xff, 0xff, 0xc0, + 0xce, 0xf2, 0x80, 0x7d, 0x02, 0x00, 0x00, +} From 288667f436bca9a579fcd4626e8017c4d9d86e89 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Tue, 13 Nov 2018 18:01:18 -0800 Subject: [PATCH 3/4] add KubeletPodResources feature-gate --- pkg/features/kube_features.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 7a739cede3312..849de58a00884 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -386,6 +386,12 @@ const ( // // Allow TTL controller to clean up Pods and Jobs after they finish. TTLAfterFinished utilfeature.Feature = "TTLAfterFinished" + + // owner: @dashpole + // alpha: v1.13 + // + // Enables the kubelet's pod resources grpc endpoint + KubeletPodResources utilfeature.Feature = "KubeletPodResources" ) func init() { @@ -453,6 +459,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS VolumeSnapshotDataSource: {Default: false, PreRelease: utilfeature.Alpha}, ProcMountType: {Default: false, PreRelease: utilfeature.Alpha}, TTLAfterFinished: {Default: false, PreRelease: utilfeature.Alpha}, + KubeletPodResources: {Default: false, PreRelease: utilfeature.Alpha}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: From 630cb53f82c19d0acc3116f7d48196f329ab6d10 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Tue, 13 Nov 2018 19:25:56 -0800 Subject: [PATCH 4/4] add kubelet grpc server for pod-resources service --- cmd/kubelet/app/server.go | 3 + pkg/kubelet/BUILD | 1 + pkg/kubelet/apis/BUILD | 1 + pkg/kubelet/apis/podresources/BUILD | 48 ++++++ pkg/kubelet/apis/podresources/client.go | 44 ++++++ pkg/kubelet/apis/podresources/constants.go | 22 +++ pkg/kubelet/apis/podresources/server.go | 75 +++++++++ pkg/kubelet/apis/podresources/server_test.go | 153 +++++++++++++++++++ pkg/kubelet/cm/BUILD | 1 + pkg/kubelet/cm/container_manager.go | 4 + pkg/kubelet/cm/container_manager_linux.go | 5 + pkg/kubelet/cm/container_manager_stub.go | 5 + pkg/kubelet/cm/container_manager_windows.go | 5 + pkg/kubelet/cm/devicemanager/BUILD | 1 + pkg/kubelet/cm/devicemanager/manager.go | 8 + pkg/kubelet/cm/devicemanager/manager_stub.go | 6 + pkg/kubelet/cm/devicemanager/pod_devices.go | 19 +++ pkg/kubelet/cm/devicemanager/types.go | 4 + pkg/kubelet/config/defaults.go | 1 + pkg/kubelet/kubelet.go | 12 ++ pkg/kubelet/kubelet_getters.go | 5 + pkg/kubelet/server/BUILD | 4 + pkg/kubelet/server/server.go | 15 ++ pkg/kubelet/util/util_unix.go | 10 ++ pkg/kubelet/util/util_unsupported.go | 5 + pkg/kubelet/util/util_windows.go | 9 ++ test/e2e_node/BUILD | 4 + test/e2e_node/device_plugin.go | 15 ++ test/e2e_node/util.go | 24 +++ 29 files changed, 509 insertions(+) create mode 100644 pkg/kubelet/apis/podresources/BUILD create mode 100644 pkg/kubelet/apis/podresources/client.go create mode 100644 pkg/kubelet/apis/podresources/constants.go create mode 100644 pkg/kubelet/apis/podresources/server.go create mode 100644 pkg/kubelet/apis/podresources/server_test.go diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index 9a324ee1d4b35..04422bb899095 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -1015,6 +1015,9 @@ func startKubelet(k kubelet.Bootstrap, podCfg *config.PodConfig, kubeCfg *kubele if kubeCfg.ReadOnlyPort > 0 { go k.ListenAndServeReadOnly(net.ParseIP(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort)) } + if utilfeature.DefaultFeatureGate.Enabled(features.KubeletPodResources) { + go k.ListenAndServePodResources() + } } func CreateAndInitKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration, diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index 74d34494a98a0..182e53517c957 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -47,6 +47,7 @@ go_library( "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/apis/pluginregistration/v1:go_default_library", + "//pkg/kubelet/apis/podresources:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/certificate:go_default_library", "//pkg/kubelet/checkpointmanager:go_default_library", diff --git a/pkg/kubelet/apis/BUILD b/pkg/kubelet/apis/BUILD index f90b69ce06a78..f253ed6bc2664 100644 --- a/pkg/kubelet/apis/BUILD +++ b/pkg/kubelet/apis/BUILD @@ -40,6 +40,7 @@ filegroup( "//pkg/kubelet/apis/pluginregistration/v1:all-srcs", "//pkg/kubelet/apis/pluginregistration/v1alpha1:all-srcs", "//pkg/kubelet/apis/pluginregistration/v1beta1:all-srcs", + "//pkg/kubelet/apis/podresources:all-srcs", "//pkg/kubelet/apis/stats/v1alpha1:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/kubelet/apis/podresources/BUILD b/pkg/kubelet/apis/podresources/BUILD new file mode 100644 index 0000000000000..8bca4b897022b --- /dev/null +++ b/pkg/kubelet/apis/podresources/BUILD @@ -0,0 +1,48 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "client.go", + "constants.go", + "server.go", + ], + importpath = "k8s.io/kubernetes/pkg/kubelet/apis/podresources", + visibility = ["//visibility:public"], + deps = [ + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", + "//pkg/kubelet/util:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["server_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", + "//staging/src/k8s.io/api/core/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//vendor/github.com/stretchr/testify/mock:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//pkg/kubelet/apis/podresources/v1alpha1:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/pkg/kubelet/apis/podresources/client.go b/pkg/kubelet/apis/podresources/client.go new file mode 100644 index 0000000000000..cb3ca222ed3ec --- /dev/null +++ b/pkg/kubelet/apis/podresources/client.go @@ -0,0 +1,44 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/util" +) + +// GetClient returns a client for the PodResourcesLister grpc service +func GetClient(socket string, connectionTimeout time.Duration, maxMsgSize int) (podresourcesapi.PodResourcesListerClient, *grpc.ClientConn, error) { + addr, dialer, err := util.GetAddressAndDialer(socket) + if err != nil { + return nil, nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), connectionTimeout) + defer cancel() + + conn, err := grpc.DialContext(ctx, addr, grpc.WithInsecure(), grpc.WithDialer(dialer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize))) + if err != nil { + return nil, nil, fmt.Errorf("Error dialing socket %s: %v", socket, err) + } + return podresourcesapi.NewPodResourcesListerClient(conn), conn, nil +} diff --git a/pkg/kubelet/apis/podresources/constants.go b/pkg/kubelet/apis/podresources/constants.go new file mode 100644 index 0000000000000..6cc4c6a261a3b --- /dev/null +++ b/pkg/kubelet/apis/podresources/constants.go @@ -0,0 +1,22 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +const ( + // Socket is the name of the podresources server socket + Socket = "kubelet" +) diff --git a/pkg/kubelet/apis/podresources/server.go b/pkg/kubelet/apis/podresources/server.go new file mode 100644 index 0000000000000..f39e2b26ce0cb --- /dev/null +++ b/pkg/kubelet/apis/podresources/server.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +import ( + "context" + + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" +) + +// DevicesProvider knows how to provide the devices used by the given container +type DevicesProvider interface { + GetDevices(podUID, containerName string) []*v1alpha1.ContainerDevices +} + +// PodsProvider knows how to provide the pods admitted by the node +type PodsProvider interface { + GetPods() []*v1.Pod +} + +// podResourcesServer implements PodResourcesListerServer +type podResourcesServer struct { + podsProvider PodsProvider + devicesProvider DevicesProvider +} + +// NewPodResourcesServer returns a PodResourcesListerServer which lists pods provided by the PodsProvider +// with device information provided by the DevicesProvider +func NewPodResourcesServer(podsProvider PodsProvider, devicesProvider DevicesProvider) v1alpha1.PodResourcesListerServer { + return &podResourcesServer{ + podsProvider: podsProvider, + devicesProvider: devicesProvider, + } +} + +// List returns information about the resources assigned to pods on the node +func (p *podResourcesServer) List(ctx context.Context, req *v1alpha1.ListPodResourcesRequest) (*v1alpha1.ListPodResourcesResponse, error) { + pods := p.podsProvider.GetPods() + podResources := make([]*v1alpha1.PodResources, len(pods)) + + for i, pod := range pods { + pRes := v1alpha1.PodResources{ + Name: pod.Name, + Namespace: pod.Namespace, + Containers: make([]*v1alpha1.ContainerResources, len(pod.Spec.Containers)), + } + + for j, container := range pod.Spec.Containers { + pRes.Containers[j] = &v1alpha1.ContainerResources{ + Name: container.Name, + Devices: p.devicesProvider.GetDevices(string(pod.UID), container.Name), + } + } + podResources[i] = &pRes + } + + return &v1alpha1.ListPodResourcesResponse{ + PodResources: podResources, + }, nil +} diff --git a/pkg/kubelet/apis/podresources/server_test.go b/pkg/kubelet/apis/podresources/server_test.go new file mode 100644 index 0000000000000..60c14d4c954a5 --- /dev/null +++ b/pkg/kubelet/apis/podresources/server_test.go @@ -0,0 +1,153 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podresources + +import ( + "context" + "testing" + + "github.com/stretchr/testify/mock" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" +) + +type mockProvider struct { + mock.Mock +} + +func (m *mockProvider) GetPods() []*v1.Pod { + args := m.Called() + return args.Get(0).([]*v1.Pod) +} + +func (m *mockProvider) GetDevices(podUID, containerName string) []*v1alpha1.ContainerDevices { + args := m.Called(podUID, containerName) + return args.Get(0).([]*v1alpha1.ContainerDevices) +} + +func TestListPodResources(t *testing.T) { + podName := "pod-name" + podNamespace := "pod-namespace" + podUID := types.UID("pod-uid") + containerName := "container-name" + + devs := []*v1alpha1.ContainerDevices{ + { + ResourceName: "resource", + DeviceIds: []string{"dev0", "dev1"}, + }, + } + + for _, tc := range []struct { + desc string + pods []*v1.Pod + devices []*v1alpha1.ContainerDevices + expectedResponse *v1alpha1.ListPodResourcesResponse + }{ + { + desc: "no pods", + pods: []*v1.Pod{}, + devices: []*v1alpha1.ContainerDevices{}, + expectedResponse: &v1alpha1.ListPodResourcesResponse{}, + }, + { + desc: "pod without devices", + pods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + UID: podUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: containerName, + }, + }, + }, + }, + }, + devices: []*v1alpha1.ContainerDevices{}, + expectedResponse: &v1alpha1.ListPodResourcesResponse{ + PodResources: []*v1alpha1.PodResources{ + { + Name: podName, + Namespace: podNamespace, + Containers: []*v1alpha1.ContainerResources{ + { + Name: containerName, + Devices: []*v1alpha1.ContainerDevices{}, + }, + }, + }, + }, + }, + }, + { + desc: "pod with devices", + pods: []*v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: podNamespace, + UID: podUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: containerName, + }, + }, + }, + }, + }, + devices: devs, + expectedResponse: &v1alpha1.ListPodResourcesResponse{ + PodResources: []*v1alpha1.PodResources{ + { + Name: podName, + Namespace: podNamespace, + Containers: []*v1alpha1.ContainerResources{ + { + Name: containerName, + Devices: devs, + }, + }, + }, + }, + }, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + m := new(mockProvider) + m.On("GetPods").Return(tc.pods) + m.On("GetDevices", string(podUID), containerName).Return(tc.devices) + server := NewPodResourcesServer(m, m) + resp, err := server.List(context.TODO(), &v1alpha1.ListPodResourcesRequest{}) + if err != nil { + t.Errorf("want err = %v, got %q", nil, err) + } + if tc.expectedResponse.String() != resp.String() { + t.Errorf("want resp = %s, got %s", tc.expectedResponse.String(), resp.String()) + } + }) + } +} diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 12e67142dc8c0..6ead9ea8c3dd1 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -27,6 +27,7 @@ go_library( deps = [ "//pkg/features:go_default_library", "//pkg/kubelet/apis/cri:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/cm/cpumanager:go_default_library", "//pkg/kubelet/config:go_default_library", "//pkg/kubelet/container:go_default_library", diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 64728a7d6b1fc..a9fe926ee0459 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -23,6 +23,7 @@ import ( // TODO: Migrate kubelet to either use its own internal objects or client library. "k8s.io/api/core/v1" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" @@ -100,6 +101,9 @@ type ContainerManager interface { // The pluginwatcher's Handlers allow to have a single module for handling // registration. GetPluginRegistrationHandler() pluginwatcher.PluginHandler + + // GetDevices returns information about the devices assigned to pods and containers + GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices } type NodeConfig struct { diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index 8253df72bd15e..9432fc71adc47 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -44,6 +44,7 @@ import ( "k8s.io/client-go/tools/record" kubefeatures "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager" @@ -878,3 +879,7 @@ func (cm *containerManagerImpl) GetCapacity() v1.ResourceList { func (cm *containerManagerImpl) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) { return cm.deviceManager.GetCapacity() } + +func (cm *containerManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices { + return cm.deviceManager.GetDevices(podUID, containerName) +} diff --git a/pkg/kubelet/cm/container_manager_stub.go b/pkg/kubelet/cm/container_manager_stub.go index 47fdd617bf074..4563dc530489a 100644 --- a/pkg/kubelet/cm/container_manager_stub.go +++ b/pkg/kubelet/cm/container_manager_stub.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -105,6 +106,10 @@ func (cm *containerManagerStub) GetPodCgroupRoot() string { return "" } +func (cm *containerManagerStub) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { + return nil +} + func NewStubContainerManager() ContainerManager { return &containerManagerStub{} } diff --git a/pkg/kubelet/cm/container_manager_windows.go b/pkg/kubelet/cm/container_manager_windows.go index 409875eb33453..a8a84d5f22de7 100644 --- a/pkg/kubelet/cm/container_manager_windows.go +++ b/pkg/kubelet/cm/container_manager_windows.go @@ -31,6 +31,7 @@ import ( "k8s.io/klog" kubefeatures "k8s.io/kubernetes/pkg/features" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" "k8s.io/kubernetes/pkg/kubelet/config" @@ -166,3 +167,7 @@ func (cm *containerManagerImpl) InternalContainerLifecycle() InternalContainerLi func (cm *containerManagerImpl) GetPodCgroupRoot() string { return "" } + +func (cm *containerManagerImpl) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { + return nil +} diff --git a/pkg/kubelet/cm/devicemanager/BUILD b/pkg/kubelet/cm/devicemanager/BUILD index 9a14643b71c08..54a67ec3f97a1 100644 --- a/pkg/kubelet/cm/devicemanager/BUILD +++ b/pkg/kubelet/cm/devicemanager/BUILD @@ -16,6 +16,7 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", "//pkg/kubelet/apis/pluginregistration/v1alpha1:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/checkpointmanager/errors:go_default_library", "//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library", diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go index 2a326aba9f8b9..5434aa29f8ec1 100644 --- a/pkg/kubelet/cm/devicemanager/manager.go +++ b/pkg/kubelet/cm/devicemanager/manager.go @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" @@ -802,3 +803,10 @@ func (m *ManagerImpl) isDevicePluginResource(resource string) bool { } return false } + +// GetDevices returns the devices used by the specified container +func (m *ManagerImpl) GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices { + m.mutex.Lock() + defer m.mutex.Unlock() + return m.podDevices.getContainerDevices(podUID, containerName) +} diff --git a/pkg/kubelet/cm/devicemanager/manager_stub.go b/pkg/kubelet/cm/devicemanager/manager_stub.go index 1008daca3b7c2..e32b671ffb20e 100644 --- a/pkg/kubelet/cm/devicemanager/manager_stub.go +++ b/pkg/kubelet/cm/devicemanager/manager_stub.go @@ -18,6 +18,7 @@ package devicemanager import ( "k8s.io/api/core/v1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" @@ -61,3 +62,8 @@ func (h *ManagerStub) GetCapacity() (v1.ResourceList, v1.ResourceList, []string) func (h *ManagerStub) GetWatcherHandler() pluginwatcher.PluginHandler { return nil } + +// GetDevices returns nil +func (h *ManagerStub) GetDevices(_, _ string) []*podresourcesapi.ContainerDevices { + return nil +} diff --git a/pkg/kubelet/cm/devicemanager/pod_devices.go b/pkg/kubelet/cm/devicemanager/pod_devices.go index d3d0cc00b1ecb..4fd4b196c4edb 100644 --- a/pkg/kubelet/cm/devicemanager/pod_devices.go +++ b/pkg/kubelet/cm/devicemanager/pod_devices.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" ) @@ -271,3 +272,21 @@ func (pdev podDevices) deviceRunContainerOptions(podUID, contName string) *Devic } return opts } + +// getContainerDevices returns the devices assigned to the provided container for all ResourceNames +func (pdev podDevices) getContainerDevices(podUID, contName string) []*podresourcesapi.ContainerDevices { + if _, podExists := pdev[podUID]; !podExists { + return nil + } + if _, contExists := pdev[podUID][contName]; !contExists { + return nil + } + cDev := []*podresourcesapi.ContainerDevices{} + for resource, allocateInfo := range pdev[podUID][contName] { + cDev = append(cDev, &podresourcesapi.ContainerDevices{ + ResourceName: resource, + DeviceIds: allocateInfo.deviceIds.UnsortedList(), + }) + } + return cDev +} diff --git a/pkg/kubelet/cm/devicemanager/types.go b/pkg/kubelet/cm/devicemanager/types.go index 35923b00d127f..8396378b4074d 100644 --- a/pkg/kubelet/cm/devicemanager/types.go +++ b/pkg/kubelet/cm/devicemanager/types.go @@ -20,6 +20,7 @@ import ( "time" "k8s.io/api/core/v1" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/config" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/lifecycle" @@ -54,6 +55,9 @@ type Manager interface { // and inactive device plugin resources previously registered on the node. GetCapacity() (v1.ResourceList, v1.ResourceList, []string) GetWatcherHandler() watcher.PluginHandler + + // GetDevices returns information about the devices assigned to pods and containers + GetDevices(podUID, containerName string) []*podresourcesapi.ContainerDevices } // DeviceRunContainerOptions contains the combined container runtime settings to consume its allocated devices. diff --git a/pkg/kubelet/config/defaults.go b/pkg/kubelet/config/defaults.go index b90306968f1fa..16bd6cb56491c 100644 --- a/pkg/kubelet/config/defaults.go +++ b/pkg/kubelet/config/defaults.go @@ -23,4 +23,5 @@ const ( DefaultKubeletPluginsDirName = "plugins" DefaultKubeletContainersDirName = "containers" DefaultKubeletPluginContainersDirName = "plugin-containers" + DefaultKubeletPodResourcesDirName = "pod-resources" ) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 404b0a148be14..0e065a1b6290d 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -60,6 +60,7 @@ import ( kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/config" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" pluginwatcherapi "k8s.io/kubernetes/pkg/kubelet/apis/pluginregistration/v1" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" "k8s.io/kubernetes/pkg/kubelet/cadvisor" kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" @@ -97,6 +98,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/sysctl" "k8s.io/kubernetes/pkg/kubelet/token" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/kubelet/util/manager" "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" @@ -192,6 +194,7 @@ type Bootstrap interface { StartGarbageCollection() ListenAndServe(address net.IP, port uint, tlsOptions *server.TLSOptions, auth server.AuthInterface, enableDebuggingHandlers, enableContentionProfiling bool) ListenAndServeReadOnly(address net.IP, port uint) + ListenAndServePodResources() Run(<-chan kubetypes.PodUpdate) RunOnce(<-chan kubetypes.PodUpdate) ([]RunPodResult, error) } @@ -1242,6 +1245,7 @@ func allGlobalUnicastIPs() ([]net.IP, error) { // 1. the root directory // 2. the pods directory // 3. the plugins directory +// 4. the pod-resources directory func (kl *Kubelet) setupDataDirs() error { kl.rootDirectory = path.Clean(kl.rootDirectory) if err := os.MkdirAll(kl.getRootDir(), 0750); err != nil { @@ -1256,6 +1260,9 @@ func (kl *Kubelet) setupDataDirs() error { if err := os.MkdirAll(kl.getPluginsDir(), 0750); err != nil { return fmt.Errorf("error creating plugins directory: %v", err) } + if err := os.MkdirAll(kl.getPodResourcesDir(), 0750); err != nil { + return fmt.Errorf("error creating podresources directory: %v", err) + } return nil } @@ -2221,6 +2228,11 @@ func (kl *Kubelet) ListenAndServeReadOnly(address net.IP, port uint) { server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, address, port) } +// ListenAndServePodResources runs the kubelet podresources grpc service +func (kl *Kubelet) ListenAndServePodResources() { + server.ListenAndServePodResources(util.LocalEndpoint(kl.getPodResourcesDir(), podresources.Socket), kl.podManager, kl.containerManager) +} + // Delete the eligible dead container instances in a pod. Depending on the configuration, the latest dead containers may be kept around. func (kl *Kubelet) cleanUpContainersInPod(podID types.UID, exitedContainerID string) { if podStatus, err := kl.podCache.Get(podID); err == nil { diff --git a/pkg/kubelet/kubelet_getters.go b/pkg/kubelet/kubelet_getters.go index 7256adb1e162e..3e6bc43e9b36e 100644 --- a/pkg/kubelet/kubelet_getters.go +++ b/pkg/kubelet/kubelet_getters.go @@ -139,6 +139,11 @@ func (kl *Kubelet) getPodContainerDir(podUID types.UID, ctrName string) string { return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletContainersDirName, ctrName) } +// getPodResourcesSocket returns the full path to the directory containing the pod resources socket +func (kl *Kubelet) getPodResourcesDir() string { + return filepath.Join(kl.getRootDir(), config.DefaultKubeletPodResourcesDirName) +} + // GetPods returns all pods bound to the kubelet and their spec, and the mirror // pods. func (kl *Kubelet) GetPods() []*v1.Pod { diff --git a/pkg/kubelet/server/BUILD b/pkg/kubelet/server/BUILD index 103e166d3b5ec..595bca6af35e5 100644 --- a/pkg/kubelet/server/BUILD +++ b/pkg/kubelet/server/BUILD @@ -18,6 +18,8 @@ go_library( "//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1/validation:go_default_library", + "//pkg/kubelet/apis/podresources:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/prober:go_default_library", "//pkg/kubelet/server/portforward:go_default_library", @@ -25,6 +27,7 @@ go_library( "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/server/streaming:go_default_library", "//pkg/kubelet/types:go_default_library", + "//pkg/kubelet/util:go_default_library", "//pkg/util/configz:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -47,6 +50,7 @@ go_library( "//vendor/github.com/google/cadvisor/metrics:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/github.com/prometheus/client_golang/prometheus/promhttp:go_default_library", + "//vendor/google.golang.org/grpc:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/pkg/kubelet/server/server.go b/pkg/kubelet/server/server.go index 141ed74cbfac8..66bc7f54d9586 100644 --- a/pkg/kubelet/server/server.go +++ b/pkg/kubelet/server/server.go @@ -37,6 +37,7 @@ import ( "github.com/google/cadvisor/metrics" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "google.golang.org/grpc" "k8s.io/klog" "k8s.io/api/core/v1" @@ -56,6 +57,8 @@ import ( "k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/core/v1/validation" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober" "k8s.io/kubernetes/pkg/kubelet/server/portforward" @@ -63,6 +66,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/kubelet/server/streaming" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/pkg/util/configz" ) @@ -161,6 +165,17 @@ func ListenAndServeKubeletReadOnlyServer(host HostInterface, resourceAnalyzer st klog.Fatal(server.ListenAndServe()) } +// ListenAndServePodResources initializes a grpc server to serve the PodResources service +func ListenAndServePodResources(socket string, podsProvider podresources.PodsProvider, devicesProvider podresources.DevicesProvider) { + server := grpc.NewServer() + podresourcesapi.RegisterPodResourcesListerServer(server, podresources.NewPodResourcesServer(podsProvider, devicesProvider)) + l, err := util.CreateListener(socket) + if err != nil { + klog.Fatalf("Failed to create listener for podResources endpoint: %v", err) + } + klog.Fatal(server.Serve(l)) +} + // AuthInterface contains all methods required by the auth filters type AuthInterface interface { authenticator.Request diff --git a/pkg/kubelet/util/util_unix.go b/pkg/kubelet/util/util_unix.go index fe4483095be93..0ddc638439146 100644 --- a/pkg/kubelet/util/util_unix.go +++ b/pkg/kubelet/util/util_unix.go @@ -23,6 +23,7 @@ import ( "net" "net/url" "os" + "path/filepath" "time" "golang.org/x/sys/unix" @@ -99,3 +100,12 @@ func parseEndpoint(endpoint string) (string, string, error) { return u.Scheme, "", fmt.Errorf("protocol %q not supported", u.Scheme) } } + +// LocalEndpoint returns the full path to a unix socket at the given endpoint +func LocalEndpoint(path, file string) string { + u := url.URL{ + Scheme: unixProtocol, + Path: path, + } + return filepath.Join(u.String(), file+".sock") +} diff --git a/pkg/kubelet/util/util_unsupported.go b/pkg/kubelet/util/util_unsupported.go index 77f14ea5255a3..6661678acedf9 100644 --- a/pkg/kubelet/util/util_unsupported.go +++ b/pkg/kubelet/util/util_unsupported.go @@ -40,3 +40,8 @@ func LockAndCheckSubPath(volumePath, subPath string) ([]uintptr, error) { // UnlockPath empty implementation func UnlockPath(fileHandles []uintptr) { } + +// LocalEndpoint empty implementation +func LocalEndpoint(path, file string) string { + return "" +} diff --git a/pkg/kubelet/util/util_windows.go b/pkg/kubelet/util/util_windows.go index cac80755a7342..7123728ff94ab 100644 --- a/pkg/kubelet/util/util_windows.go +++ b/pkg/kubelet/util/util_windows.go @@ -103,3 +103,12 @@ func parseEndpoint(endpoint string) (string, string, error) { return u.Scheme, "", fmt.Errorf("protocol %q not supported", u.Scheme) } } + +// LocalEndpoint returns the full path to a windows named pipe +func LocalEndpoint(path, file string) string { + u := url.URL{ + Scheme: npipeProtocol, + Path: path, + } + return u.String() + "//./pipe/" + file +} diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 6f7856eb4a59d..51c902c0e3535 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -24,12 +24,15 @@ go_library( "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/apis/deviceplugin/v1beta1:go_default_library", + "//pkg/kubelet/apis/podresources:go_default_library", + "//pkg/kubelet/apis/podresources/v1alpha1:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/kubeletconfig/util/codec:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/remote:go_default_library", + "//pkg/kubelet/util:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", @@ -49,6 +52,7 @@ go_library( "//vendor/github.com/onsi/ginkgo:go_default_library", "//vendor/github.com/onsi/gomega:go_default_library", "//vendor/github.com/prometheus/common/model:go_default_library", + "//vendor/golang.org/x/net/context:go_default_library", "//vendor/k8s.io/klog:go_default_library", ] + select({ "@io_bazel_rules_go//go/platform:linux": [ diff --git a/test/e2e_node/device_plugin.go b/test/e2e_node/device_plugin.go index 62c8fc0749706..3951f8c538b2a 100644 --- a/test/e2e_node/device_plugin.go +++ b/test/e2e_node/device_plugin.go @@ -60,7 +60,11 @@ func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSo Context("DevicePlugin", func() { By("Enabling support for Kubelet Plugins Watcher") tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) { + if initialConfig.FeatureGates == nil { + initialConfig.FeatureGates = map[string]bool{} + } initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher + initialConfig.FeatureGates[string(features.KubeletPodResources)] = true }) It("Verifies the Kubelet device plugin functionality.", func() { By("Start stub device plugin") @@ -98,6 +102,17 @@ func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSo devId1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE) Expect(devId1).To(Not(Equal(""))) + podResources, err := getNodeDevices() + Expect(err).To(BeNil()) + Expect(len(podResources.PodResources)).To(Equal(1)) + Expect(podResources.PodResources[0].Name).To(Equal(pod1.Name)) + Expect(podResources.PodResources[0].Namespace).To(Equal(pod1.Namespace)) + Expect(len(podResources.PodResources[0].Containers)).To(Equal(1)) + Expect(podResources.PodResources[0].Containers[0].Name).To(Equal(pod1.Spec.Containers[0].Name)) + Expect(len(podResources.PodResources[0].Containers[0].Devices)).To(Equal(1)) + Expect(podResources.PodResources[0].Containers[0].Devices[0].ResourceName).To(Equal(resourceName)) + Expect(len(podResources.PodResources[0].Containers[0].Devices[0].DeviceIds)).To(Equal(1)) + pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{}) framework.ExpectNoError(err) diff --git a/test/e2e_node/util.go b/test/e2e_node/util.go index eae0e6a7e6e14..fd6353e90b2f2 100644 --- a/test/e2e_node/util.go +++ b/test/e2e_node/util.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "golang.org/x/net/context" "k8s.io/klog" apiv1 "k8s.io/api/core/v1" @@ -38,11 +39,14 @@ import ( "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cm" kubeletconfigcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec" kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/remote" + "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/metrics" frameworkmetrics "k8s.io/kubernetes/test/e2e/framework/metrics" @@ -62,6 +66,10 @@ var busyboxImage = imageutils.GetE2EImage(imageutils.BusyBox) const ( // Kubelet internal cgroup name for node allocatable cgroup. defaultNodeAllocatableCgroup = "kubepods" + // defaultPodResourcesPath is the path to the local endpoint serving the podresources GRPC service. + defaultPodResourcesPath = "/var/lib/kubelet/pod-resources" + defaultPodResourcesTimeout = 10 * time.Second + defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb ) func getNodeSummary() (*stats.Summary, error) { @@ -92,6 +100,22 @@ func getNodeSummary() (*stats.Summary, error) { return &summary, nil } +func getNodeDevices() (*podresourcesapi.ListPodResourcesResponse, error) { + endpoint := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + client, conn, err := podresources.GetClient(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + if err != nil { + return nil, fmt.Errorf("Error getting grpc client: %v", err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.List(ctx, &podresourcesapi.ListPodResourcesRequest{}) + if err != nil { + return nil, fmt.Errorf("%v.Get(_) = _, %v", client, err) + } + return resp, nil +} + // Returns the current KubeletConfiguration func getCurrentKubeletConfig() (*kubeletconfig.KubeletConfiguration, error) { resp := pollConfigz(5*time.Minute, 5*time.Second)